Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bd4e46458c |
+4
-1
@@ -50,8 +50,11 @@ MOLECULE_ENV=development # Environment label (development/
|
||||
# Container/runtime detection
|
||||
# MOLECULE_IN_DOCKER= # Set when running the platform inside Docker (accepts 1/0, true/false). Triggers A2A proxy to rewrite 127.0.0.1:<port> agent URLs to Docker bridge hostnames. Auto-detected via /.dockerenv; only set if detection fails or to force off.
|
||||
|
||||
# Observability (Awareness)
|
||||
# AWARENESS_URL= # If set, injected into workspace containers along with a deterministic AWARENESS_NAMESPACE derived from workspace ID. Enables the cross-session memory MCP server.
|
||||
|
||||
# GitHub
|
||||
# GITHUB_REPO=owner/repo # Target repo for agent initial_prompt clone (e.g. Molecule-AI/molecule-core). Read inside workspace containers.
|
||||
# GITHUB_REPO=owner/repo # Target repo for agent initial_prompt clone (e.g. Molecule-AI/molecule-monorepo). Read inside workspace containers.
|
||||
# GITHUB_TOKEN= # Personal access token / installation token used by agents that clone private repos. Register as a global secret via POST /admin/secrets for propagation to workspace env. Token is used in-URL during clone and then scrubbed from .git/config via `git remote set-url`.
|
||||
|
||||
# Webhooks
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
refire:1778784369
|
||||
@@ -18,24 +18,15 @@
|
||||
# per §SOP-6 security model). No-op when merged=false.
|
||||
#
|
||||
# Required env (set by the workflow):
|
||||
# GITEA_TOKEN, GITEA_HOST, REPO, PR_NUMBER
|
||||
# plus one of REQUIRED_CHECKS_JSON (preferred) or REQUIRED_CHECKS (legacy)
|
||||
# GITEA_TOKEN, GITEA_HOST, REPO, PR_NUMBER, REQUIRED_CHECKS
|
||||
#
|
||||
# REQUIRED_CHECKS_JSON is a JSON object keyed by branch name. Each value
|
||||
# is an array of status-check context names that branch protection
|
||||
# requires for that branch. The script looks up the PR's base branch and
|
||||
# evaluates only the checks declared for that branch.
|
||||
#
|
||||
# {"main": ["CI / all-required (pull_request)", ...],
|
||||
# "staging": ["CI / all-required (pull_request)", ...]}
|
||||
#
|
||||
# REQUIRED_CHECKS (legacy) is a newline-separated list used when the
|
||||
# JSON variable is not set. Declared in the workflow YAML rather than
|
||||
# fetched from /branch_protections (which needs admin scope — sop-tier-bot
|
||||
# has read-only). Trade dynamism for simplicity: when the required-check
|
||||
# set changes, update both branch protection AND this env. Keeping them
|
||||
# in sync is less complexity than granting the audit bot admin perms on
|
||||
# every repo.
|
||||
# REQUIRED_CHECKS is a newline-separated list of status-check context
|
||||
# names that branch protection requires. Declared in the workflow YAML
|
||||
# rather than fetched from /branch_protections (which needs admin
|
||||
# scope — sop-tier-bot has read-only). Trade dynamism for simplicity:
|
||||
# when the required-check set changes, update both branch protection
|
||||
# AND this env. Keeping them in sync is less complexity than granting
|
||||
# the audit bot admin perms on every repo.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -43,10 +34,7 @@ set -euo pipefail
|
||||
: "${GITEA_HOST:?required}"
|
||||
: "${REPO:?required}"
|
||||
: "${PR_NUMBER:?required}"
|
||||
if [ -z "${REQUIRED_CHECKS_JSON:-}" ] && [ -z "${REQUIRED_CHECKS:-}" ]; then
|
||||
echo "::error::Either REQUIRED_CHECKS_JSON or REQUIRED_CHECKS must be set"
|
||||
exit 1
|
||||
fi
|
||||
: "${REQUIRED_CHECKS:?required (newline-separated context names)}"
|
||||
|
||||
OWNER="${REPO%%/*}"
|
||||
NAME="${REPO##*/}"
|
||||
@@ -61,30 +49,21 @@ if [ "$MERGED" != "true" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# NOTE: no || true — with set -euo pipefail, jq parse failures (e.g. field
|
||||
# missing from API response) propagate as hard errors. Use jq's // operator
|
||||
# for graceful defaults instead of bash || true guards. This was re-added by
|
||||
# 8c343e3a ("fix(gitea): add || true guards to jq pipelines") — reverted
|
||||
# here because the guards mask silent failures that hide malformed API responses.
|
||||
MERGE_SHA=$(echo "$PR" | jq -r '.merge_commit_sha // empty')
|
||||
MERGED_BY=$(echo "$PR" | jq -r '.merged_by.login // "unknown"')
|
||||
TITLE=$(echo "$PR" | jq -r '.title // ""')
|
||||
BASE_BRANCH=$(echo "$PR" | jq -r '.base.ref // "main"')
|
||||
HEAD_SHA=$(echo "$PR" | jq -r '.head.sha // empty')
|
||||
MERGE_SHA=$(echo "$PR" | jq -r '.merge_commit_sha // empty') || true
|
||||
MERGED_BY=$(echo "$PR" | jq -r '.merged_by.login // "unknown"') || true
|
||||
TITLE=$(echo "$PR" | jq -r '.title // ""') || true
|
||||
BASE_BRANCH=$(echo "$PR" | jq -r '.base.ref // "main"') || true
|
||||
HEAD_SHA=$(echo "$PR" | jq -r '.head.sha // empty') || true
|
||||
|
||||
if [ -z "$MERGE_SHA" ]; then
|
||||
echo "::warning::PR #${PR_NUMBER} merged=true but no merge_commit_sha — cannot evaluate force-merge."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 2. Required status checks — branch-aware JSON dict takes precedence.
|
||||
if [ -n "${REQUIRED_CHECKS_JSON:-}" ]; then
|
||||
REQUIRED=$(echo "$REQUIRED_CHECKS_JSON" | jq -r --arg branch "$BASE_BRANCH" '.[$branch] // [] | .[]')
|
||||
else
|
||||
REQUIRED="$REQUIRED_CHECKS"
|
||||
fi
|
||||
# 2. Required status checks declared in the workflow env.
|
||||
REQUIRED="$REQUIRED_CHECKS"
|
||||
if [ -z "${REQUIRED//[[:space:]]/}" ]; then
|
||||
echo "::notice::REQUIRED_CHECKS empty for branch '$BASE_BRANCH' — force-merge not applicable."
|
||||
echo "::notice::REQUIRED_CHECKS empty — force-merge not applicable."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -96,7 +75,7 @@ STATUS=$(curl -sS -H "$AUTH" \
|
||||
declare -A CHECK_STATE
|
||||
while IFS=$'\t' read -r ctx state; do
|
||||
[ -n "$ctx" ] && CHECK_STATE[$ctx]="$state"
|
||||
done < <(echo "$STATUS" | jq -r '.statuses // [] | .[] | "\(.context)\t\(.status)"')
|
||||
done < <(echo "$STATUS" | jq -r '.statuses // [] | .[] | "\(.context)\t\(.status)"') || true
|
||||
|
||||
# 4. For each required check, was it green at merge? YAML block scalars
|
||||
# (`|`) leave a trailing newline; skip blank/whitespace-only lines.
|
||||
@@ -118,10 +97,7 @@ fi
|
||||
|
||||
# 5. Emit structured audit event.
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
# jq -R (raw input) converts each line to a JSON string; jq -s wraps into array.
|
||||
# If FAILED_CHECKS is unexpectedly empty (shouldn't happen — we exit above),
|
||||
# this produces []. No || true needed.
|
||||
FAILED_JSON=$(printf '%s\n' "${FAILED_CHECKS[@]}" | jq -R . | jq -s .)
|
||||
FAILED_JSON=$(printf '%s\n' "${FAILED_CHECKS[@]}" | jq -R . | jq -s .) || true
|
||||
|
||||
# Print as a single-line JSON so Vector's parse_json transform can pick
|
||||
# it up cleanly from docker_logs.
|
||||
|
||||
@@ -203,17 +203,12 @@ def ci_jobs_all(ci_doc: dict) -> set[str]:
|
||||
|
||||
def ci_job_names(ci_doc: dict) -> set[str]:
|
||||
"""Set of job keys in ci.yml MINUS the sentinel itself MINUS jobs
|
||||
whose `if:` gates on `github.event_name` or `github.ref` (those are
|
||||
event-scoped and can legitimately be `skipped` for a given trigger;
|
||||
if we required them under the sentinel `needs:`, every PR-only job
|
||||
whose `if:` gates on `github.event_name` (those are event-scoped
|
||||
and can legitimately be `skipped` for a given trigger; if we
|
||||
required them under the sentinel `needs:`, every PR-only job
|
||||
would be `skipped` on push and the sentinel would interpret
|
||||
`skipped != success` as failure). RFC §4 spec.
|
||||
|
||||
`github.ref` is the companion gate for jobs that run only on direct
|
||||
pushes to specific branches (e.g. `github.ref == 'refs/heads/main'`).
|
||||
These never execute in a PR context, so flagging them as missing
|
||||
from `all-required.needs:` is a false positive (mc#958 / mc#959).
|
||||
|
||||
Used for F1 (jobs missing from sentinel needs). NOT used for F1b
|
||||
(typos in needs) — see `ci_jobs_all` for that."""
|
||||
jobs = ci_doc.get("jobs")
|
||||
@@ -226,9 +221,7 @@ def ci_job_names(ci_doc: dict) -> set[str]:
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
gate = v.get("if")
|
||||
if isinstance(gate, str) and (
|
||||
"github.event_name" in gate or "github.ref" in gate
|
||||
):
|
||||
if isinstance(gate, str) and "github.event_name" in gate:
|
||||
continue
|
||||
names.add(k)
|
||||
return names
|
||||
@@ -274,8 +267,7 @@ def required_checks_env(audit_doc: dict) -> set[str]:
|
||||
found.append(v)
|
||||
if not found:
|
||||
sys.stderr.write(
|
||||
f"::error::REQUIRED_CHECKS env not found in any step of "
|
||||
f"{AUDIT_WORKFLOW_PATH}\n"
|
||||
f"::error::REQUIRED_CHECKS env not found in any step of {AUDIT_WORKFLOW_PATH}\n"
|
||||
)
|
||||
sys.exit(3)
|
||||
if len(found) > 1:
|
||||
@@ -385,15 +377,10 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
contexts = set(protection.get("status_check_contexts") or [])
|
||||
|
||||
# ----- F1: job exists in CI but not under sentinel.needs -----
|
||||
# Post-#1766 contract: the sentinel may deliberately have no `needs:`
|
||||
# and instead poll path-relevant statuses dynamically. In that case
|
||||
# F1 is a false positive — skip it. F1b (typos in existing needs)
|
||||
# is naturally skipped when needs is empty.
|
||||
missing_from_needs = sorted(jobs - needs)
|
||||
if missing_from_needs and needs:
|
||||
if missing_from_needs:
|
||||
findings.append(
|
||||
"F1 — jobs in ci.yml NOT under sentinel `needs:` "
|
||||
"(sentinel doesn't gate them):\n"
|
||||
"F1 — jobs in ci.yml NOT under sentinel `needs:` (sentinel doesn't gate them):\n"
|
||||
+ "\n".join(f" - {n}" for n in missing_from_needs)
|
||||
)
|
||||
|
||||
@@ -403,8 +390,7 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
stale_needs = sorted(needs - jobs_all)
|
||||
if stale_needs:
|
||||
findings.append(
|
||||
"F1b — sentinel `needs:` lists jobs NOT present in ci.yml "
|
||||
"(typo or removed job):\n"
|
||||
"F1b — sentinel `needs:` lists jobs NOT present in ci.yml (typo or removed job):\n"
|
||||
+ "\n".join(f" - {n}" for n in stale_needs)
|
||||
)
|
||||
|
||||
@@ -412,9 +398,7 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
# Compute the contexts the CI YAML actually produces. The sentinel
|
||||
# is in (B) intentionally (`ci / all-required (pull_request)`); we
|
||||
# whitelist it explicitly.
|
||||
emitted_contexts = {
|
||||
expected_context(j) for j in jobs
|
||||
} | {expected_context(SENTINEL_JOB)}
|
||||
emitted_contexts = {expected_context(j) for j in jobs} | {expected_context(SENTINEL_JOB)}
|
||||
# Contexts NOT produced by ci.yml may still come from other
|
||||
# workflows in the repo (Secret scan etc). We can't enumerate
|
||||
# every workflow's emissions cheaply; instead, flag only contexts
|
||||
@@ -427,9 +411,8 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
)
|
||||
if stale_protection:
|
||||
findings.append(
|
||||
"F2 — protection `status_check_contexts` entries with `ci / ` "
|
||||
"prefix that NO job in ci.yml emits "
|
||||
"(stale name → silent advisory gate):\n"
|
||||
"F2 — protection `status_check_contexts` entries with `ci / ` prefix that NO "
|
||||
"job in ci.yml emits (stale name → silent advisory gate):\n"
|
||||
+ "\n".join(f" - {c}" for c in stale_protection)
|
||||
)
|
||||
|
||||
@@ -504,8 +487,7 @@ def render_body(branch: str, findings: list[str], debug: dict) -> str:
|
||||
f"# Drift detected on `{REPO}/{branch}`",
|
||||
"",
|
||||
"Auto-filed by `.gitea/workflows/ci-required-drift.yml` "
|
||||
"(RFC [internal#219]"
|
||||
"(https://git.moleculesai.app/molecule-ai/internal/issues/219) §4 + §6).",
|
||||
"(RFC [internal#219](https://git.moleculesai.app/molecule-ai/internal/issues/219) §4 + §6).",
|
||||
"",
|
||||
"## Findings",
|
||||
"",
|
||||
@@ -516,11 +498,8 @@ def render_body(branch: str, findings: list[str], debug: dict) -> str:
|
||||
"",
|
||||
"## Resolution",
|
||||
"",
|
||||
"- **F1 / F1b**: if the sentinel job has a `needs:` block, add "
|
||||
"the missing job to it in `.gitea/workflows/ci.yml`, or remove "
|
||||
"the stale entry. If the sentinel deliberately has no `needs:` "
|
||||
"(path-aware polling sentinel per post-#1766 contract), this "
|
||||
"finding is expected and F1 is skipped.",
|
||||
"- **F1 / F1b**: add the missing job to `all-required.needs:` "
|
||||
"in `.gitea/workflows/ci.yml`, or remove the stale entry.",
|
||||
"- **F2**: rename the protection context to match an emitter, "
|
||||
"or remove it from `status_check_contexts` "
|
||||
"(PATCH `/api/v1/repos/{owner}/{repo}/branch_protections/{branch}`).",
|
||||
@@ -561,12 +540,12 @@ def file_or_update(
|
||||
|
||||
if dry_run:
|
||||
print(f"::notice::[dry-run] would file/update drift issue for {branch}")
|
||||
print("::group::[dry-run] title")
|
||||
print(f"::group::[dry-run] title")
|
||||
print(title)
|
||||
print("::endgroup::")
|
||||
print("::group::[dry-run] body")
|
||||
print(f"::endgroup::")
|
||||
print(f"::group::[dry-run] body")
|
||||
print(body)
|
||||
print("::endgroup::")
|
||||
print(f"::endgroup::")
|
||||
return
|
||||
|
||||
existing = find_open_issue(title)
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared path-filter helper for Gitea Actions workflows.
|
||||
|
||||
Computes changed files against the PR base SHA or push-before SHA and writes
|
||||
boolean outputs to GITHUB_OUTPUT. If the diff base is missing or untrusted, the
|
||||
helper fails open by setting every output in the selected profile to true.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROFILES: dict[str, dict[str, str]] = {
|
||||
"ci": {
|
||||
"platform": r"^workspace-server/",
|
||||
"canvas": r"^canvas/",
|
||||
"python": r"^workspace/",
|
||||
"scripts": r"^tests/e2e/|^scripts/|^infra/scripts/",
|
||||
},
|
||||
"handlers-postgres": {
|
||||
"handlers": (
|
||||
r"^workspace-server/internal/handlers/"
|
||||
r"|^workspace-server/internal/wsauth/"
|
||||
r"|^workspace-server/migrations/"
|
||||
r"|^\.gitea/workflows/handlers-postgres-integration\.yml$"
|
||||
),
|
||||
},
|
||||
"e2e-api": {
|
||||
"api": r"^workspace-server/|^tests/e2e/|^\.gitea/workflows/e2e-api\.yml$",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def classify(profile: str, paths: list[str]) -> dict[str, bool]:
|
||||
patterns = PROFILES[profile]
|
||||
return {
|
||||
name: any(re.search(pattern, path) for path in paths)
|
||||
for name, pattern in patterns.items()
|
||||
}
|
||||
|
||||
|
||||
def all_true(profile: str) -> dict[str, bool]:
|
||||
return {name: True for name in PROFILES[profile]}
|
||||
|
||||
|
||||
def resolve_base(event_name: str, pr_base_sha: str, push_before: str) -> str:
|
||||
if event_name == "pull_request" and pr_base_sha:
|
||||
return pr_base_sha
|
||||
return push_before
|
||||
|
||||
|
||||
def is_zero_sha(value: str) -> bool:
|
||||
return not value or bool(re.fullmatch(r"0+", value))
|
||||
|
||||
|
||||
def run_git(args: list[str], *, timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def base_exists(base: str) -> bool:
|
||||
return run_git(["cat-file", "-e", base]).returncode == 0
|
||||
|
||||
|
||||
def fetch_base(base: str, base_ref: str) -> None:
|
||||
# Gitea may reject fetching an arbitrary unadvertised SHA from a shallow
|
||||
# PR checkout. Fetch the advertised base branch first, then fall back to
|
||||
# the SHA for hosts that allow it.
|
||||
if base_ref:
|
||||
run_git(["fetch", "--depth=1", "origin", base_ref])
|
||||
if not base_exists(base):
|
||||
run_git(["fetch", "--depth=1", "origin", base])
|
||||
|
||||
|
||||
def deepen_base_ref(base_ref: str) -> None:
|
||||
if base_ref:
|
||||
run_git(["fetch", "--deepen=200", "origin", base_ref], timeout=60)
|
||||
|
||||
|
||||
def merge_base(base: str) -> str | None:
|
||||
proc = run_git(["merge-base", base, "HEAD"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
value = proc.stdout.strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def changed_paths(base: str, *, use_merge_base: bool) -> list[str] | None:
|
||||
compare_base = base
|
||||
if use_merge_base:
|
||||
compare_base = merge_base(base) or ""
|
||||
if not compare_base:
|
||||
return None
|
||||
|
||||
proc = run_git(["diff", "--name-only", compare_base, "HEAD"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return [line for line in proc.stdout.splitlines() if line]
|
||||
|
||||
|
||||
def write_outputs(values: dict[str, bool], output_path: str | None) -> None:
|
||||
lines = [f"{name}={'true' if value else 'false'}" for name, value in values.items()]
|
||||
if output_path:
|
||||
with Path(output_path).open("a", encoding="utf-8") as fh:
|
||||
for line in lines:
|
||||
fh.write(line + "\n")
|
||||
else:
|
||||
for line in lines:
|
||||
print(line)
|
||||
|
||||
|
||||
def detect(
|
||||
profile: str,
|
||||
event_name: str,
|
||||
pr_base_sha: str,
|
||||
push_before: str,
|
||||
base_ref: str = "",
|
||||
) -> dict[str, bool]:
|
||||
base = resolve_base(event_name, pr_base_sha, push_before)
|
||||
if is_zero_sha(base):
|
||||
return all_true(profile)
|
||||
|
||||
if not base_exists(base):
|
||||
fetch_base(base, base_ref)
|
||||
if not base_exists(base):
|
||||
return all_true(profile)
|
||||
|
||||
use_merge_base = event_name == "pull_request"
|
||||
if use_merge_base and base_ref and merge_base(base) is None:
|
||||
deepen_base_ref(base_ref)
|
||||
|
||||
paths = changed_paths(base, use_merge_base=use_merge_base)
|
||||
if paths is None:
|
||||
return all_true(profile)
|
||||
return classify(profile, paths)
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--profile", required=True, choices=sorted(PROFILES))
|
||||
parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME", ""))
|
||||
parser.add_argument("--pr-base-sha", default="")
|
||||
parser.add_argument("--base-ref", default="")
|
||||
parser.add_argument(
|
||||
"--push-before",
|
||||
default=os.environ.get("GITHUB_EVENT_BEFORE", ""),
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
values = detect(
|
||||
args.profile,
|
||||
args.event_name,
|
||||
args.pr_base_sha,
|
||||
args.push_before,
|
||||
args.base_ref,
|
||||
)
|
||||
write_outputs(values, os.environ.get("GITHUB_OUTPUT"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -1,503 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""gitea-merge-queue — conservative serialized merge bot for Gitea.
|
||||
|
||||
Gitea 1.22.6 has auto-merge (`pull_auto_merge`) but no GitHub-style merge
|
||||
queue. This script provides the missing serialized policy in user space:
|
||||
|
||||
1. Pick the oldest open PR carrying QUEUE_LABEL.
|
||||
2. Refuse to act unless main is green.
|
||||
3. Refuse fork PRs; the queue may only mutate same-repo branches.
|
||||
4. If the PR branch does not contain current main, call Gitea's
|
||||
/pulls/{n}/update endpoint and stop. CI must rerun on the updated head.
|
||||
5. If the updated PR head has all required contexts green, merge with the
|
||||
non-bypass merge actor token.
|
||||
|
||||
The script is intentionally one-PR-per-run. Workflow/cron concurrency should
|
||||
serialize invocations so two green PRs cannot merge against the same main.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _env(key: str, *, default: str = "") -> str:
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
GITEA_TOKEN = _env("GITEA_TOKEN")
|
||||
GITEA_HOST = _env("GITEA_HOST")
|
||||
REPO = _env("REPO")
|
||||
WATCH_BRANCH = _env("WATCH_BRANCH", default="main")
|
||||
QUEUE_LABEL = _env("QUEUE_LABEL", default="merge-queue")
|
||||
HOLD_LABEL = _env("HOLD_LABEL", default="merge-queue-hold")
|
||||
UPDATE_STYLE = _env("UPDATE_STYLE", default="merge")
|
||||
REQUIRED_CONTEXTS_RAW = _env(
|
||||
"REQUIRED_CONTEXTS",
|
||||
default=(
|
||||
"CI / all-required (pull_request),"
|
||||
"sop-checklist / all-items-acked (pull_request)"
|
||||
),
|
||||
)
|
||||
# Required contexts for push (main/staging) runs. The push CI uses the same
|
||||
# aggregator names with " (push)" suffix. Checking these explicitly instead of
|
||||
# the combined state avoids false-pause when non-blocking jobs (e.g. Platform
|
||||
# Go with continue-on-error: true due to mc#774) have failed — their failures
|
||||
# pollute the combined state but do not block merges.
|
||||
PUSH_REQUIRED_CONTEXTS_RAW = _env(
|
||||
"PUSH_REQUIRED_CONTEXTS",
|
||||
default="CI / all-required (push)",
|
||||
)
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
|
||||
|
||||
class ApiError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class MergePermissionError(ApiError):
|
||||
"""Merge failed with a permanent permission error (403/404/405).
|
||||
The queue should skip this PR and move to the next one."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class MergeDecision:
|
||||
ready: bool
|
||||
action: str
|
||||
reason: str
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "WATCH_BRANCH", "QUEUE_LABEL"):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
if UPDATE_STYLE not in {"merge", "rebase"}:
|
||||
sys.stderr.write("::error::UPDATE_STYLE must be merge or rebase\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read()
|
||||
status = exc.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(f"{method} {path} -> HTTP {status}: {snippet}")
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
if expect_json:
|
||||
raise ApiError(f"{method} {path} -> HTTP {status} non-JSON: {exc}") from exc
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
def required_contexts(raw: str) -> list[str]:
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
def push_required_contexts() -> list[str]:
|
||||
"""Required contexts for push (branch) CI runs. See PUSH_REQUIRED_CONTEXTS_RAW."""
|
||||
return required_contexts(PUSH_REQUIRED_CONTEXTS_RAW)
|
||||
|
||||
|
||||
def status_state(status: dict) -> str:
|
||||
return str(status.get("status") or status.get("state") or "").lower()
|
||||
|
||||
|
||||
def latest_statuses_by_context(statuses: list[dict]) -> dict[str, dict]:
|
||||
# Gitea /statuses endpoint returns entries in ascending id order (oldest
|
||||
# first). We need the LAST occurrence of each context, so iterate in
|
||||
# reverse to prefer newer entries.
|
||||
latest: dict[str, dict] = {}
|
||||
for status in reversed(statuses):
|
||||
context = status.get("context")
|
||||
if isinstance(context, str):
|
||||
latest[context] = status # overwrite: reverse order → newest wins
|
||||
return latest
|
||||
|
||||
|
||||
def _is_tier_low_pending_ok(
|
||||
latest_statuses: dict[str, dict],
|
||||
context: str,
|
||||
pr_labels: set[str],
|
||||
) -> bool:
|
||||
"""Return True if tier:low PR can tolerate sop-checklist pending state.
|
||||
|
||||
Per sop-checklist-config.yaml tier_failure_mode, tier:low uses soft-fail:
|
||||
sop-checklist posts state=pending when acks are satisfied (missing
|
||||
manager/ceo acks are informational only). The queue should accept
|
||||
pending instead of waiting for success.
|
||||
"""
|
||||
if "tier:low" not in pr_labels:
|
||||
return False
|
||||
if "sop-checklist" not in context:
|
||||
return False
|
||||
status = latest_statuses.get(context) or {}
|
||||
return status_state(status) == "pending"
|
||||
|
||||
|
||||
def required_contexts_green(
|
||||
latest_statuses: dict[str, dict],
|
||||
contexts: list[str],
|
||||
pr_labels: set[str] | None = None,
|
||||
) -> tuple[bool, list[str]]:
|
||||
missing_or_bad: list[str] = []
|
||||
for context in contexts:
|
||||
status = latest_statuses.get(context)
|
||||
state = status_state(status or {})
|
||||
if state != "success":
|
||||
if pr_labels and _is_tier_low_pending_ok(
|
||||
latest_statuses, context, pr_labels
|
||||
):
|
||||
continue # tier:low soft-fail: accept pending sop-checklist
|
||||
missing_or_bad.append(f"{context}={state or 'missing'}")
|
||||
return not missing_or_bad, missing_or_bad
|
||||
|
||||
|
||||
def label_names(issue: dict) -> set[str]:
|
||||
return {
|
||||
label["name"]
|
||||
for label in issue.get("labels", [])
|
||||
if isinstance(label, dict) and isinstance(label.get("name"), str)
|
||||
}
|
||||
|
||||
|
||||
def choose_next_queued_issue(
|
||||
issues: list[dict],
|
||||
*,
|
||||
queue_label: str,
|
||||
hold_label: str = "",
|
||||
) -> dict | None:
|
||||
candidates = []
|
||||
for issue in issues:
|
||||
labels = label_names(issue)
|
||||
if queue_label not in labels:
|
||||
continue
|
||||
if hold_label and hold_label in labels:
|
||||
continue
|
||||
if "pull_request" not in issue:
|
||||
continue
|
||||
candidates.append(issue)
|
||||
candidates.sort(key=lambda issue: (issue.get("created_at") or "", int(issue["number"])))
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def pr_contains_base_sha(commits: list[dict], base_sha: str) -> bool:
|
||||
for commit in commits:
|
||||
sha = commit.get("sha") or commit.get("id")
|
||||
if sha == base_sha:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def pr_has_current_base(pr: dict, commits: list[dict], main_sha: str) -> bool:
|
||||
if pr.get("merge_base") == main_sha:
|
||||
return True
|
||||
return pr_contains_base_sha(commits, main_sha)
|
||||
|
||||
|
||||
def evaluate_merge_readiness(
|
||||
*,
|
||||
main_status: dict,
|
||||
pr_status: dict,
|
||||
required_contexts: list[str],
|
||||
pr_has_current_base: bool,
|
||||
pr_labels: set[str] | None = None,
|
||||
) -> MergeDecision:
|
||||
# Check push-required contexts explicitly instead of combined state.
|
||||
# Combined state can be "failure" due to non-blocking jobs
|
||||
# (continue-on-error: true) that don't actually gate merges.
|
||||
# CI / all-required (push) is the authoritative gate — it respects
|
||||
# continue-on-error and correctly aggregates all blocking failures.
|
||||
main_latest = latest_statuses_by_context(main_status.get("statuses") or [])
|
||||
main_ok, main_bad = required_contexts_green(main_latest, push_required_contexts())
|
||||
if not main_ok:
|
||||
return MergeDecision(False, "pause", "main required contexts not green: " + ", ".join(main_bad))
|
||||
if not pr_has_current_base:
|
||||
return MergeDecision(False, "update", "PR head does not contain current main")
|
||||
|
||||
# Check explicit required contexts instead of combined state. Combined state
|
||||
# can be "failure" due to non-blocking jobs with continue-on-error: true
|
||||
# (e.g. publish-runtime-autobump/pr-validate, qa-review on stale tokens).
|
||||
# The required_contexts list is the authoritative gate — it includes only
|
||||
# the checks that actually block merges.
|
||||
latest = latest_statuses_by_context(pr_status.get("statuses") or [])
|
||||
ok, missing_or_bad = required_contexts_green(latest, required_contexts, pr_labels)
|
||||
if not ok:
|
||||
return MergeDecision(False, "wait", "required contexts not green: " + ", ".join(missing_or_bad))
|
||||
return MergeDecision(True, "merge", "ready")
|
||||
|
||||
|
||||
def get_branch_head(branch: str) -> str:
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/branches/{branch}")
|
||||
commit = body.get("commit") if isinstance(body, dict) else None
|
||||
sha = commit.get("id") if isinstance(commit, dict) else None
|
||||
if not isinstance(sha, str) or len(sha) < 7:
|
||||
raise ApiError(f"branch {branch} response missing commit id")
|
||||
return sha
|
||||
|
||||
|
||||
def get_combined_status(sha: str) -> dict:
|
||||
"""Combined status + all individual statuses for `sha`.
|
||||
|
||||
The /status endpoint caps the `statuses` array at 30 entries (Gitea
|
||||
default page size), so we fetch the full list via /statuses with a
|
||||
higher limit. The combined `state` still comes from /status.
|
||||
"""
|
||||
_, combined = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status")
|
||||
if not isinstance(combined, dict):
|
||||
raise ApiError(f"status for {sha} response not object")
|
||||
combined_statuses: list[dict] = combined.get("statuses") or []
|
||||
try:
|
||||
_, all_statuses_raw = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/commits/{sha}/statuses",
|
||||
query={"limit": "50"},
|
||||
)
|
||||
if isinstance(all_statuses_raw, list):
|
||||
all_statuses: list[dict] = list(all_statuses_raw)
|
||||
else:
|
||||
all_statuses = []
|
||||
except (ApiError, urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
sys.stderr.write(f"::warning::could not fetch full statuses list for {sha[:8]}: {exc}\n")
|
||||
all_statuses = []
|
||||
# Build latest per context: process combined (ascending→reverse=newest
|
||||
# first), then fill gaps from all_statuses (already newest-first).
|
||||
latest: dict[str, dict] = {}
|
||||
for status in reversed(sorted(combined_statuses, key=lambda s: s.get("id") or 0)):
|
||||
ctx = status.get("context")
|
||||
if isinstance(ctx, str) and ctx not in latest:
|
||||
latest[ctx] = status
|
||||
for status in all_statuses:
|
||||
ctx = status.get("context")
|
||||
if isinstance(ctx, str) and ctx not in latest:
|
||||
latest[ctx] = status
|
||||
combined["statuses"] = list(latest.values())
|
||||
return combined
|
||||
|
||||
|
||||
def list_queued_issues() -> list[dict]:
|
||||
_, body = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={
|
||||
"state": "open",
|
||||
"type": "pulls",
|
||||
"labels": QUEUE_LABEL,
|
||||
"limit": "50",
|
||||
},
|
||||
)
|
||||
if not isinstance(body, list):
|
||||
raise ApiError("queued issues response not list")
|
||||
return body
|
||||
|
||||
|
||||
def get_pull(pr_number: int) -> dict:
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"PR #{pr_number} response not object")
|
||||
return body
|
||||
|
||||
|
||||
def get_pull_commits(pr_number: int) -> list[dict]:
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/commits")
|
||||
if not isinstance(body, list):
|
||||
raise ApiError(f"PR #{pr_number} commits response not list")
|
||||
return body
|
||||
|
||||
|
||||
def post_comment(pr_number: int, body: str, *, dry_run: bool) -> None:
|
||||
print(f"::notice::comment PR #{pr_number}: {body.splitlines()[0][:160]}")
|
||||
if dry_run:
|
||||
return
|
||||
api("POST", f"/repos/{OWNER}/{NAME}/issues/{pr_number}/comments", body={"body": body})
|
||||
|
||||
|
||||
def update_pull(pr_number: int, *, dry_run: bool) -> None:
|
||||
print(f"::notice::updating PR #{pr_number} with base branch via style={UPDATE_STYLE}")
|
||||
if dry_run:
|
||||
return
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/update",
|
||||
query={"style": UPDATE_STYLE},
|
||||
expect_json=False,
|
||||
)
|
||||
|
||||
|
||||
def merge_pull(pr_number: int, *, dry_run: bool) -> None:
|
||||
payload = {
|
||||
"Do": "merge",
|
||||
"MergeTitleField": f"Merge PR #{pr_number} via Gitea merge queue",
|
||||
"MergeMessageField": (
|
||||
"Serialized merge by gitea-merge-queue after current-main, "
|
||||
"SOP, and required CI checks were green."
|
||||
),
|
||||
}
|
||||
print(f"::notice::merging PR #{pr_number}")
|
||||
if dry_run:
|
||||
return
|
||||
try:
|
||||
api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
|
||||
except ApiError as exc:
|
||||
# Re-raise permission-like errors so process_once can skip this PR.
|
||||
# 403 = no push access, 404 = repo/pr not found, 405 = not allowed.
|
||||
msg = str(exc)
|
||||
for code in ("403", "404", "405"):
|
||||
if code in msg:
|
||||
raise MergePermissionError(msg) from exc
|
||||
raise # re-raise other ApiErrors unchanged
|
||||
|
||||
|
||||
def process_once(*, dry_run: bool = False) -> int:
|
||||
contexts = required_contexts(REQUIRED_CONTEXTS_RAW)
|
||||
main_sha = get_branch_head(WATCH_BRANCH)
|
||||
main_status = get_combined_status(main_sha)
|
||||
# Check push-required contexts explicitly instead of combined state.
|
||||
# See evaluate_merge_readiness for rationale.
|
||||
main_latest = latest_statuses_by_context(main_status.get("statuses") or [])
|
||||
main_ok, main_bad = required_contexts_green(main_latest, push_required_contexts())
|
||||
if not main_ok:
|
||||
print(f"::notice::queue paused: {WATCH_BRANCH}@{main_sha[:8]} required contexts not green: {', '.join(main_bad)}")
|
||||
return 0
|
||||
|
||||
issue = choose_next_queued_issue(
|
||||
list_queued_issues(),
|
||||
queue_label=QUEUE_LABEL,
|
||||
hold_label=HOLD_LABEL,
|
||||
)
|
||||
if not issue:
|
||||
print("::notice::merge queue empty")
|
||||
return 0
|
||||
|
||||
pr_number = int(issue["number"])
|
||||
pr = get_pull(pr_number)
|
||||
if pr.get("state") != "open":
|
||||
print(f"::notice::PR #{pr_number} is not open; skipping")
|
||||
return 0
|
||||
if pr.get("base", {}).get("ref") != WATCH_BRANCH:
|
||||
post_comment(pr_number, f"merge-queue: skipped; base branch is not `{WATCH_BRANCH}`.", dry_run=dry_run)
|
||||
return 0
|
||||
if pr.get("head", {}).get("repo_id") != pr.get("base", {}).get("repo_id"):
|
||||
post_comment(pr_number, "merge-queue: skipped; fork PRs are not supported by the serialized queue.", dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
head_sha = pr.get("head", {}).get("sha")
|
||||
if not isinstance(head_sha, str) or len(head_sha) < 7:
|
||||
raise ApiError(f"PR #{pr_number} missing head sha")
|
||||
commits = get_pull_commits(pr_number)
|
||||
current_base = pr_has_current_base(pr, commits, main_sha)
|
||||
pr_status = get_combined_status(head_sha)
|
||||
pr_labels = label_names(pr)
|
||||
decision = evaluate_merge_readiness(
|
||||
main_status=main_status,
|
||||
pr_status=pr_status,
|
||||
required_contexts=contexts,
|
||||
pr_has_current_base=current_base,
|
||||
pr_labels=pr_labels,
|
||||
)
|
||||
|
||||
print(f"::notice::PR #{pr_number} decision={decision.action}: {decision.reason}")
|
||||
if decision.action == "update":
|
||||
update_pull(pr_number, dry_run=dry_run)
|
||||
post_comment(
|
||||
pr_number,
|
||||
(
|
||||
f"merge-queue: updated this branch with `{WATCH_BRANCH}` at "
|
||||
f"`{main_sha[:12]}`. Waiting for CI on the refreshed head."
|
||||
),
|
||||
dry_run=dry_run,
|
||||
)
|
||||
return 0
|
||||
if decision.ready:
|
||||
latest_main_sha = get_branch_head(WATCH_BRANCH)
|
||||
if latest_main_sha != main_sha:
|
||||
print(
|
||||
f"::notice::main moved {main_sha[:8]} -> {latest_main_sha[:8]}; "
|
||||
"deferring to next tick"
|
||||
)
|
||||
return 0
|
||||
try:
|
||||
merge_pull(pr_number, dry_run=dry_run)
|
||||
except MergePermissionError as exc:
|
||||
# Permanent merge failure (HTTP 403/404/405). Post a comment so
|
||||
# maintainers know why, then return 0 so this tick is done.
|
||||
# The PR stays in the queue; future ticks can retry after the
|
||||
# permission issue is resolved.
|
||||
sys.stderr.write(f"::error::merge permission error for PR #{pr_number}: {exc}\n")
|
||||
post_comment(
|
||||
pr_number,
|
||||
(
|
||||
"merge-queue: merge failed with HTTP 405 'User not allowed to merge PR'. "
|
||||
"No available token has Can-merge permission on this repo. "
|
||||
"Fix: grant Can-merge to a token, or add a maintain/admin collaborator. "
|
||||
"Skipping to next queued PR on next tick."
|
||||
),
|
||||
dry_run=dry_run,
|
||||
)
|
||||
return 0
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
_require_runtime_env()
|
||||
try:
|
||||
return process_once(dry_run=args.dry_run)
|
||||
except ApiError as exc:
|
||||
# API errors (401/403/404/500) are transient for a queue tick —
|
||||
# log and exit 0 so the workflow is not marked failed and the next
|
||||
# tick can retry. Returning non-zero would permanently fail the
|
||||
# workflow run, blocking future ticks.
|
||||
sys.stderr.write(f"::error::queue API error: {exc}\n")
|
||||
return 0
|
||||
except urllib.error.URLError as exc:
|
||||
sys.stderr.write(f"::error::queue network error: {exc}\n")
|
||||
return 0
|
||||
except TimeoutError as exc:
|
||||
sys.stderr.write(f"::error::queue timeout: {exc}\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,111 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lint workflow bash for curl status-code capture pollution.
|
||||
|
||||
The bad shape is:
|
||||
|
||||
HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
|
||||
|
||||
`curl -w` writes the HTTP code to stdout before returning non-zero, so
|
||||
fallback output inside the same command substitution appends another code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
SELF = ".gitea/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
|
||||
class Finding(NamedTuple):
|
||||
path: str
|
||||
snippet: str
|
||||
|
||||
|
||||
BAD_STATUS_CAPTURE = re.compile(
|
||||
r"""
|
||||
\$\(\s*
|
||||
curl\b
|
||||
[^)]*
|
||||
-w\s*['"]%\{http_code\}['"]
|
||||
[^)]*
|
||||
\|\|\s*
|
||||
(?:
|
||||
echo\s+['"]?000['"]?
|
||||
|
|
||||
printf\s+['"]000['"]
|
||||
)
|
||||
\s*\)
|
||||
""",
|
||||
re.DOTALL | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _logical_shell(content: str) -> str:
|
||||
"""Collapse bash line continuations so one curl command is one string."""
|
||||
return re.sub(r"\\\s*\n\s*", " ", content)
|
||||
|
||||
|
||||
def scan_content(path: str, content: str) -> list[Finding]:
|
||||
flat = _logical_shell(content)
|
||||
return [
|
||||
Finding(path=path, snippet=re.sub(r"\s+", " ", match.group(0)).strip()[:160])
|
||||
for match in BAD_STATUS_CAPTURE.finditer(flat)
|
||||
]
|
||||
|
||||
|
||||
def scan_paths(paths: list[str]) -> list[Finding]:
|
||||
findings: list[Finding] = []
|
||||
for path in paths:
|
||||
if path == SELF:
|
||||
continue
|
||||
content = Path(path).read_text(encoding="utf-8")
|
||||
findings.extend(scan_content(path, content))
|
||||
return findings
|
||||
|
||||
|
||||
def default_paths() -> list[str]:
|
||||
return sorted(glob.glob(".gitea/workflows/*.yml"))
|
||||
|
||||
|
||||
def print_report(findings: list[Finding]) -> None:
|
||||
if not findings:
|
||||
print("OK No curl-status-capture pollution patterns detected")
|
||||
return
|
||||
|
||||
print(f"::error::Found {len(findings)} curl-status-capture pollution site(s):")
|
||||
for finding in findings:
|
||||
print(
|
||||
f"::error file={finding.path}::Curl status-capture pollution: "
|
||||
"'|| echo/printf 000' inside a $(curl ... -w '%{http_code}' ...) "
|
||||
"subshell. On non-2xx or connection failure, curl's -w writes a "
|
||||
"status, then exits non-zero, then the fallback appends another "
|
||||
"status. Fix: route -w into a tempfile so the exit code cannot "
|
||||
"pollute stdout."
|
||||
)
|
||||
print(f" matched: {finding.snippet}...")
|
||||
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(" set +e")
|
||||
print(" curl ... -w '%{http_code}' >code.txt 2>/dev/null")
|
||||
print(" set -e")
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("paths", nargs="*", help="workflow files to scan")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
paths = args.paths or default_paths()
|
||||
findings = scan_paths(paths)
|
||||
print_report(findings)
|
||||
return 1 if findings else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -29,16 +29,6 @@ Rules (4 fatal + 1 fatal cross-file + 1 heuristic-warn):
|
||||
or `https://github.com/.../releases/download` without a
|
||||
workflow-level `env.GITHUB_SERVER_URL` set to the Gitea instance.
|
||||
Memory: feedback_act_runner_github_server_url.
|
||||
7. Production deploy/redeploy workflows may not rely on Gitea
|
||||
`concurrency.cancel-in-progress: false` for serialization. Gitea
|
||||
1.22.6 can cancel queued runs despite that setting.
|
||||
8. Production deploy/redeploy workflows may not dump raw CP responses or
|
||||
raw `.error` fields into CI logs/summaries.
|
||||
9. Production deploy/redeploy workflows must expose an operational control:
|
||||
kill switch for auto deploys or rollback tag for manual deploys.
|
||||
10. Docker health checks must not run `docker info | head` under pipefail.
|
||||
`head` closes the pipe early, `docker info` can exit nonzero from
|
||||
SIGPIPE, and the step can falsely report Docker daemon failure.
|
||||
|
||||
Per `feedback_smoke_test_vendor_truth_not_shape_match`: fixtures used to
|
||||
validate this lint must mirror real Gitea 1.22.6 YAML semantics, not
|
||||
@@ -228,24 +218,6 @@ def _iter_uses(doc: Any) -> Iterable[str]:
|
||||
yield step["uses"]
|
||||
|
||||
|
||||
def _iter_run_blocks(doc: Any) -> Iterable[str]:
|
||||
"""Yield every shell `run:` block from job steps in a workflow document."""
|
||||
if not isinstance(doc, dict):
|
||||
return
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return
|
||||
for job in jobs.values():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
steps = job.get("steps")
|
||||
if not isinstance(steps, list):
|
||||
continue
|
||||
for step in steps:
|
||||
if isinstance(step, dict) and isinstance(step.get("run"), str):
|
||||
yield step["run"]
|
||||
|
||||
|
||||
def check_cross_repo_uses(filename: str, doc: Any) -> list[str]:
|
||||
"""Return per-violation error lines for cross-repo `uses:` references."""
|
||||
errors: list[str] = []
|
||||
@@ -283,23 +255,6 @@ GITHUB_API_REF_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
PROD_CP_URL_RE = re.compile(r"https://api\.moleculesai\.app\b")
|
||||
REDEPLOY_FLEET_RE = re.compile(r"\b/cp/admin/tenants/redeploy-fleet\b")
|
||||
RUN_SETS_PIPEFAIL_RE = re.compile(r"(?m)^\s*set\s+-[^\n]*o\s+pipefail\b")
|
||||
DOCKER_INFO_HEAD_PIPE_RE = re.compile(
|
||||
r"(?m)^\s*docker\s+info\b[^\n|]*\|\s*head\b"
|
||||
)
|
||||
RAW_CP_RESPONSE_RE = re.compile(
|
||||
r"""(?x)
|
||||
(?:\bjq\s+\.\s+["']?\$HTTP_RESPONSE["']?)
|
||||
|
|
||||
(?:\bcat\s+["']?\$HTTP_RESPONSE["']?)
|
||||
|
|
||||
(?:\|\s*\.error\b)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _has_workflow_level_server_url(doc: Any) -> bool:
|
||||
if not isinstance(doc, dict):
|
||||
return False
|
||||
@@ -331,107 +286,6 @@ def check_github_server_url_missing(filename: str, doc: Any, raw: str) -> list[s
|
||||
return warns
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 7-9 — production CI/CD hardening rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _is_production_redeploy_workflow(raw: str) -> bool:
|
||||
"""Heuristic production-side-effect detector.
|
||||
|
||||
We intentionally key on the production CP host plus the redeploy-fleet
|
||||
endpoint. Staging workflows call the same endpoint on staging-api and are
|
||||
governed by looser staging verification policy.
|
||||
"""
|
||||
|
||||
return bool(PROD_CP_URL_RE.search(raw) and REDEPLOY_FLEET_RE.search(raw))
|
||||
|
||||
|
||||
def _iter_concurrency_blocks(doc: Any) -> Iterable[dict[str, Any]]:
|
||||
if not isinstance(doc, dict):
|
||||
return
|
||||
top = doc.get("concurrency")
|
||||
if isinstance(top, dict):
|
||||
yield top
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return
|
||||
for job in jobs.values():
|
||||
if isinstance(job, dict) and isinstance(job.get("concurrency"), dict):
|
||||
yield job["concurrency"]
|
||||
|
||||
|
||||
def check_production_concurrency(filename: str, doc: Any, raw: str) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not _is_production_redeploy_workflow(raw):
|
||||
return errors
|
||||
for block in _iter_concurrency_blocks(doc):
|
||||
if block.get("cancel-in-progress") is False:
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 7 (FATAL): production deploy "
|
||||
f"workflow uses `concurrency.cancel-in-progress: false`. "
|
||||
f"Gitea 1.22.6 can cancel queued runs despite that setting, "
|
||||
f"so this is not a safe production serialization primitive. "
|
||||
f"Use an external queue/lock or make the deploy idempotent."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def check_production_raw_response_logging(filename: str, raw: str) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not _is_production_redeploy_workflow(raw):
|
||||
return errors
|
||||
if RAW_CP_RESPONSE_RE.search(raw):
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 8 (FATAL): production deploy "
|
||||
f"workflow appears to print a raw production CP response or raw "
|
||||
f"`.error` field. CI logs are persistent and broad-read. Redact "
|
||||
f"runtime/SSM error details; print counts, booleans, status "
|
||||
f"codes, and links to restricted observability instead."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def check_production_operational_control(filename: str, raw: str) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not _is_production_redeploy_workflow(raw):
|
||||
return errors
|
||||
has_kill_switch = "PROD_AUTO_DEPLOY_DISABLED" in raw
|
||||
has_rollback = "PROD_MANUAL_REDEPLOY_TARGET_TAG" in raw
|
||||
if not (has_kill_switch or has_rollback):
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 9 (FATAL): production deploy "
|
||||
f"workflow calls redeploy-fleet without an operational control. "
|
||||
f"Auto deploys need a `PROD_AUTO_DEPLOY_DISABLED` kill switch; "
|
||||
f"manual deploys need a `PROD_MANUAL_REDEPLOY_TARGET_TAG` "
|
||||
f"rollback/pin path."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 10 — docker info piped to head under pipefail
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_docker_info_head_pipefail(filename: str, doc: Any) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for run_block in _iter_run_blocks(doc):
|
||||
if not (
|
||||
RUN_SETS_PIPEFAIL_RE.search(run_block)
|
||||
and DOCKER_INFO_HEAD_PIPE_RE.search(run_block)
|
||||
):
|
||||
continue
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 10 (FATAL): workflow runs "
|
||||
f"`docker info | head` after enabling `pipefail`. `head` can "
|
||||
f"close the pipe early, making `docker info` exit nonzero and "
|
||||
f"falsely fail the Docker daemon health check. Capture "
|
||||
f"`docker_info=\"$(docker info 2>&1)\"` first, then print a "
|
||||
f"bounded preview with `printf ... | sed -n '1,5p'`."
|
||||
)
|
||||
break
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -482,10 +336,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
fatal_errors.extend(check_workflow_run_event(rel, doc))
|
||||
fatal_errors.extend(check_name_with_slash(rel, doc))
|
||||
fatal_errors.extend(check_cross_repo_uses(rel, doc))
|
||||
fatal_errors.extend(check_production_concurrency(rel, doc, raw))
|
||||
fatal_errors.extend(check_production_raw_response_logging(rel, raw))
|
||||
fatal_errors.extend(check_production_operational_control(rel, raw))
|
||||
fatal_errors.extend(check_docker_info_head_pipefail(rel, doc))
|
||||
warnings.extend(check_github_server_url_missing(rel, doc, raw))
|
||||
|
||||
# Cross-file checks
|
||||
|
||||
@@ -1,509 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint_bp_context_emit_match — Tier 2f per internal#350.
|
||||
|
||||
Rule
|
||||
----
|
||||
For a given protected branch, every context in
|
||||
`branch_protections/<branch>.status_check_contexts` MUST be emitted
|
||||
by at least one workflow in `.gitea/workflows/*.yml`. Two contexts
|
||||
match when:
|
||||
|
||||
1. The workflow's `name:` equals the context's workflow-part (the
|
||||
prefix before ` / `).
|
||||
2. Some job in that workflow has a `name:` (or default-fallback
|
||||
job-key) equal to the context's job-part (between ` / ` and
|
||||
` (`).
|
||||
3. The workflow's `on:` block includes the context's event-part
|
||||
(in parens at the end), with Gitea's event-name mapping:
|
||||
- `pull_request` and `pull_request_target` BOTH emit
|
||||
`(pull_request)` contexts (verified empirically on
|
||||
molecule-core/main).
|
||||
- `push` emits `(push)`.
|
||||
|
||||
A BP context with no emitter blocks merges forever — Gitea treats
|
||||
absent-as-`pending`, NOT absent-as-`skipped`-as-`success`. This is
|
||||
the phantom-required-check class
|
||||
(`feedback_phantom_required_check_after_gitea_migration`).
|
||||
|
||||
The inverse direction (emitter without BP context) is INFORMATIONAL
|
||||
only — Tier 2g handles that direction at PR-time. Flagging it here
|
||||
on a daily schedule would falsely surface every transitional state
|
||||
during a BP rollout.
|
||||
|
||||
How the gate works
|
||||
------------------
|
||||
Daily scheduled run + workflow_dispatch:
|
||||
|
||||
1. GET `branch_protections/{BRANCH}` (needs DRIFT_BOT_TOKEN with
|
||||
repo-admin scope; same persona as ci-required-drift.yml).
|
||||
Graceful-degrade on 403/404 per Tier 2a contract.
|
||||
|
||||
2. Walk `.gitea/workflows/*.yml` via PyYAML AST. For each workflow,
|
||||
enumerate its emitted contexts: `{workflow.name} / {job.name or
|
||||
job-key} ({event})` for each event in `on:` that emits a status.
|
||||
|
||||
3. For each BP context, look for an emitter match. Aggregate
|
||||
orphans.
|
||||
|
||||
4. If orphans exist:
|
||||
- File or PATCH a `[ci-bp-drift]` issue (idempotency contract:
|
||||
search for exact title prefix, edit existing if open).
|
||||
- Apply labels `tier:high` + `ci-bp-drift` (lookup IDs per
|
||||
repo; per `feedback_tier_label_ids_are_per_repo`).
|
||||
- Exit 1.
|
||||
|
||||
5. If no orphans:
|
||||
- Close any existing `[ci-bp-drift]` issue with a clean-state
|
||||
comment.
|
||||
- Exit 0.
|
||||
|
||||
Exit codes
|
||||
----------
|
||||
0 — clean OR API 403/404 (graceful-degrade, surfaces ::error::).
|
||||
1 — at least one BP context has no emitter.
|
||||
2 — env contract violation, workflows-dir missing, or YAML parse
|
||||
error.
|
||||
|
||||
Env
|
||||
---
|
||||
GITEA_TOKEN — DRIFT_BOT_TOKEN (repo-admin for branch_protections)
|
||||
GITEA_HOST — e.g. git.moleculesai.app
|
||||
REPO — owner/name
|
||||
BRANCH — defaults to `main`
|
||||
WORKFLOWS_DIR — defaults to `.gitea/workflows`
|
||||
DRIFT_LABEL — defaults to `ci-bp-drift`
|
||||
|
||||
Memory cross-links
|
||||
------------------
|
||||
- internal#350 (the RFC that specs this lint)
|
||||
- feedback_phantom_required_check_after_gitea_migration
|
||||
- feedback_tier_label_ids_are_per_repo
|
||||
- reference_post_suspension_pipeline
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
sys.stderr.write(
|
||||
"::error::PyYAML is required. Install with: pip install PyYAML\n"
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# Status-check context regex (mirrors lint-required-no-paths.py).
|
||||
_CONTEXT_RE = re.compile(
|
||||
r"^(?P<workflow>.+?) / (?P<job>.+) \((?P<event>[^)]+)\)$"
|
||||
)
|
||||
|
||||
# Map a workflow `on:` event-key to the context's event-part. Gitea's
|
||||
# emitter convention (verified on molecule-core):
|
||||
# - pull_request → `(pull_request)`
|
||||
# - pull_request_target → `(pull_request)` (same surface)
|
||||
# - push → `(push)`
|
||||
# - schedule → no PR status; scheduled runs don't post
|
||||
# commit-statuses unless the workflow itself does so explicitly.
|
||||
# - workflow_dispatch → manually dispatched runs may or may not
|
||||
# emit; safest to treat as "no PR status" (informational notice
|
||||
# only).
|
||||
_EVENT_MAP = {
|
||||
"pull_request": "pull_request",
|
||||
"pull_request_target": "pull_request",
|
||||
"push": "push",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Env
|
||||
# ---------------------------------------------------------------------------
|
||||
def _env(key: str, default: str | None = None) -> str:
|
||||
v = os.environ.get(key, default)
|
||||
return v if v is not None else ""
|
||||
|
||||
|
||||
def _require_env(key: str) -> str:
|
||||
v = os.environ.get(key)
|
||||
if not v:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API helper. Mirrors lint-required-no-paths.py's contract: returns
|
||||
# (status, payload) tuple with status ∈ {"ok", "not_found", "forbidden",
|
||||
# "error"}.
|
||||
# ---------------------------------------------------------------------------
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
) -> tuple[str, Any]:
|
||||
host = _env("GITEA_HOST")
|
||||
token = _env("GITEA_TOKEN")
|
||||
url = f"https://{host}/api/v1{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(
|
||||
url, method=method, data=data, headers=headers
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
if not raw:
|
||||
return ("ok", None)
|
||||
return ("ok", json.loads(raw))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return ("not_found", None)
|
||||
if e.code in (401, 403):
|
||||
return ("forbidden", None)
|
||||
return ("error", None)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return ("error", None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _get_on(d: Any) -> Any:
|
||||
"""YAML 1.1 boolean quirk: bare `on:` may parse to True. Handle both."""
|
||||
if not isinstance(d, dict):
|
||||
return None
|
||||
if "on" in d:
|
||||
return d["on"]
|
||||
if True in d:
|
||||
return d[True]
|
||||
return None
|
||||
|
||||
|
||||
def _on_events(doc: Any) -> set[str]:
|
||||
"""Return the set of event keys in a workflow's `on:` block.
|
||||
|
||||
Accepts all three shapes (string / list / mapping). String/list
|
||||
shapes can't carry filters but they DO emit. Returns the
|
||||
Gitea-mapped event names per `_EVENT_MAP`.
|
||||
"""
|
||||
on = _get_on(doc)
|
||||
raw_events: set[str] = set()
|
||||
if on is None:
|
||||
return raw_events
|
||||
if isinstance(on, str):
|
||||
raw_events.add(on)
|
||||
elif isinstance(on, list):
|
||||
for e in on:
|
||||
if isinstance(e, str):
|
||||
raw_events.add(e)
|
||||
elif isinstance(on, dict):
|
||||
for k in on:
|
||||
if isinstance(k, str):
|
||||
raw_events.add(k)
|
||||
return {_EVENT_MAP[e] for e in raw_events if e in _EVENT_MAP}
|
||||
|
||||
|
||||
def _job_display(jbody: dict, jkey: str) -> str:
|
||||
"""Return job's `name:` if set, else fall back to the job-key.
|
||||
|
||||
Gitea formats status contexts with the job's `name:` when set;
|
||||
when unset it uses the job key. Matches lint-required-no-paths
|
||||
convention.
|
||||
"""
|
||||
n = jbody.get("name") if isinstance(jbody, dict) else None
|
||||
if isinstance(n, str) and n:
|
||||
return n
|
||||
return jkey
|
||||
|
||||
|
||||
def workflow_contexts(doc: Any) -> set[str]:
|
||||
"""Return the set of contexts a workflow emits."""
|
||||
contexts: set[str] = set()
|
||||
if not isinstance(doc, dict):
|
||||
return contexts
|
||||
wf_name = doc.get("name")
|
||||
if not isinstance(wf_name, str) or not wf_name:
|
||||
return contexts # no name => no addressable context
|
||||
events = _on_events(doc)
|
||||
if not events:
|
||||
return contexts
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return contexts
|
||||
for jkey, jbody in jobs.items():
|
||||
if jkey == "__lines__": # tolerate line-tracking annotations
|
||||
continue
|
||||
if not isinstance(jbody, dict):
|
||||
continue
|
||||
disp = _job_display(jbody, jkey)
|
||||
for ev in events:
|
||||
contexts.add(f"{wf_name} / {disp} ({ev})")
|
||||
return contexts
|
||||
|
||||
|
||||
def parse_context(ctx: str) -> tuple[str, str, str] | None:
|
||||
m = _CONTEXT_RE.match(ctx)
|
||||
if not m:
|
||||
return None
|
||||
return (m.group("workflow"), m.group("job"), m.group("event"))
|
||||
|
||||
|
||||
def _iter_workflow_files(wf_dir: Path) -> list[Path]:
|
||||
return sorted(list(wf_dir.glob("*.yml")) + list(wf_dir.glob("*.yaml")))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Issue idempotency — search for an open issue with the canonical
|
||||
# title prefix; PATCH if found, POST if not. Mirrors ci-required-drift.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _canonical_title(repo: str, branch: str) -> str:
|
||||
return f"[ci-bp-drift] {repo}/{branch}: BP→emitter mismatch"
|
||||
|
||||
|
||||
def _ensure_labels(repo: str, names: list[str]) -> list[int]:
|
||||
status, labels = api("GET", f"/repos/{repo}/labels", query={"limit": "50"})
|
||||
if status != "ok" or not isinstance(labels, list):
|
||||
return []
|
||||
out: list[int] = []
|
||||
by_name = {label["name"]: label["id"] for label in labels if isinstance(label, dict)}
|
||||
for n in names:
|
||||
if n in by_name:
|
||||
out.append(by_name[n])
|
||||
return out
|
||||
|
||||
|
||||
def file_or_update_issue(
|
||||
repo: str, branch: str, orphans: list[str], emitter_orphans: list[str]
|
||||
) -> None:
|
||||
title = _canonical_title(repo, branch)
|
||||
body_lines = [
|
||||
f"BP→emitter drift detected on `{branch}` at "
|
||||
f"{os.environ.get('GITHUB_RUN_URL', '(run url unavailable)')}.",
|
||||
"",
|
||||
f"## Orphan BP contexts ({len(orphans)})",
|
||||
"",
|
||||
"These contexts are required by branch protection but NO workflow "
|
||||
"emits them. PRs merging into this branch will wait forever for a "
|
||||
"status that never arrives (Gitea treats absent-as-`pending`, NOT "
|
||||
"absent-as-`skipped`). See "
|
||||
"`feedback_phantom_required_check_after_gitea_migration`.",
|
||||
"",
|
||||
]
|
||||
for o in orphans:
|
||||
body_lines.append(f"- `{o}`")
|
||||
if emitter_orphans:
|
||||
body_lines += [
|
||||
"",
|
||||
f"## Workflows emitting contexts NOT in BP ({len(emitter_orphans)})",
|
||||
"",
|
||||
"Informational — Tier 2g handles this direction at PR-time. "
|
||||
"Listed here for completeness.",
|
||||
"",
|
||||
]
|
||||
for o in emitter_orphans:
|
||||
body_lines.append(f"- `{o}`")
|
||||
body_lines += [
|
||||
"",
|
||||
"Fix options:",
|
||||
" 1. PATCH `branch_protections/{branch}.status_check_contexts` "
|
||||
" to remove the orphan.",
|
||||
" 2. Restore the emitting workflow (if it was deleted/renamed).",
|
||||
"",
|
||||
"Linted by `.gitea/workflows/lint-bp-context-emit-match.yml` "
|
||||
"(Tier 2f, internal#350).",
|
||||
]
|
||||
body = "\n".join(body_lines)
|
||||
|
||||
# Idempotency search — find an open issue with the canonical title.
|
||||
status, hits = api(
|
||||
"GET",
|
||||
f"/repos/{repo}/issues",
|
||||
query={
|
||||
"type": "issues",
|
||||
"state": "open",
|
||||
"q": title,
|
||||
},
|
||||
)
|
||||
existing = None
|
||||
if status == "ok" and isinstance(hits, list):
|
||||
for h in hits:
|
||||
if (
|
||||
isinstance(h, dict)
|
||||
and h.get("state") == "open"
|
||||
and isinstance(h.get("title"), str)
|
||||
and h["title"].startswith(title)
|
||||
):
|
||||
existing = h
|
||||
break
|
||||
|
||||
label_ids = _ensure_labels(repo, ["ci-bp-drift", "tier:high"])
|
||||
|
||||
if existing:
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{repo}/issues/{existing['number']}",
|
||||
body={"body": body, "labels": label_ids} if label_ids else {"body": body},
|
||||
)
|
||||
print(
|
||||
f"::notice::Updated existing drift issue "
|
||||
f"#{existing['number']}: {existing.get('html_url', '')}"
|
||||
)
|
||||
else:
|
||||
status, posted = api(
|
||||
"POST",
|
||||
f"/repos/{repo}/issues",
|
||||
body={"title": title, "body": body, "labels": label_ids},
|
||||
)
|
||||
if status == "ok" and isinstance(posted, dict):
|
||||
print(
|
||||
f"::notice::Filed new drift issue "
|
||||
f"#{posted.get('number')}: {posted.get('html_url', '')}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
def run() -> int:
|
||||
_require_env("GITEA_TOKEN")
|
||||
_require_env("GITEA_HOST")
|
||||
repo = _require_env("REPO")
|
||||
branch = _env("BRANCH", "main")
|
||||
wf_dir = Path(_env("WORKFLOWS_DIR", ".gitea/workflows"))
|
||||
|
||||
if not wf_dir.is_dir():
|
||||
sys.stderr.write(f"::error::workflows directory not found: {wf_dir}\n")
|
||||
return 2
|
||||
|
||||
# 1. Pull BP.
|
||||
status, bp = api("GET", f"/repos/{repo}/branch_protections/{branch}")
|
||||
if status == "forbidden":
|
||||
sys.stderr.write(
|
||||
f"::error::GET branch_protections/{branch} returned HTTP 403 — "
|
||||
f"DRIFT_BOT_TOKEN lacks repo-admin scope (Gitea 1.22.6 requires "
|
||||
f"it for this endpoint). Skipping lint with exit 0 to avoid "
|
||||
f"red-X on every run. Fix: grant repo-admin to mc-drift-bot. "
|
||||
f"Per Tier 2a contract.\n"
|
||||
)
|
||||
return 0
|
||||
if status == "not_found":
|
||||
print(
|
||||
f"::notice::branch '{branch}' has no protection configured; "
|
||||
f"nothing to lint."
|
||||
)
|
||||
return 0
|
||||
if status != "ok" or not isinstance(bp, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::branch_protections/{branch} response unexpected; "
|
||||
f"status={status}. Treating as transient; exit 0.\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
bp_contexts: list[str] = list(bp.get("status_check_contexts") or [])
|
||||
if not bp_contexts:
|
||||
print(
|
||||
f"::notice::branch_protections/{branch} has 0 required "
|
||||
f"status_check_contexts; nothing to lint."
|
||||
)
|
||||
return 0
|
||||
|
||||
# 2. Enumerate emitter contexts from all workflows.
|
||||
all_emitter: set[str] = set()
|
||||
for path in _iter_workflow_files(wf_dir):
|
||||
try:
|
||||
doc = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(
|
||||
f"::error file={path}::YAML parse error: {e}; skipping.\n"
|
||||
)
|
||||
continue
|
||||
all_emitter |= workflow_contexts(doc)
|
||||
|
||||
print(
|
||||
f"::notice::Linting {len(bp_contexts)} BP context(s) for {branch} "
|
||||
f"against {len(all_emitter)} workflow-emitted context(s)."
|
||||
)
|
||||
|
||||
bp_set = set(bp_contexts)
|
||||
|
||||
# 3. Find orphans (BP-side: required but no emitter).
|
||||
bp_orphans = sorted(bp_set - all_emitter)
|
||||
|
||||
# Informational: workflow emits but BP doesn't list. Tier 2g
|
||||
# territory at PR-time. We list these as NOTICE only.
|
||||
emitter_orphans = sorted(all_emitter - bp_set)
|
||||
|
||||
if bp_orphans:
|
||||
print(
|
||||
f"::error::Found {len(bp_orphans)} BP context(s) with no "
|
||||
f"emitter — these would block merges forever (Gitea treats "
|
||||
f"absent-as-pending, not skipped):"
|
||||
)
|
||||
for o in bp_orphans:
|
||||
# Closest-match hint: name a workflow whose name-part is a
|
||||
# near-match (lev-1 typo, or same workflow with a different
|
||||
# event).
|
||||
parsed = parse_context(o)
|
||||
hint = ""
|
||||
if parsed:
|
||||
wf, _job, _ev = parsed
|
||||
candidates = sorted(
|
||||
{c for c in all_emitter if c.startswith(wf + " / ")}
|
||||
)
|
||||
if candidates:
|
||||
hint = (
|
||||
f" — closest emitter(s): {', '.join(candidates[:3])}"
|
||||
)
|
||||
print(f"::error:: - {o}{hint}")
|
||||
if emitter_orphans:
|
||||
print(
|
||||
f"::notice::Also: {len(emitter_orphans)} workflow-emitted "
|
||||
f"context(s) not in BP (informational; Tier 2g handles at "
|
||||
f"PR-time):"
|
||||
)
|
||||
for o in emitter_orphans:
|
||||
print(f"::notice:: - {o}")
|
||||
# File / patch tracking issue.
|
||||
try:
|
||||
file_or_update_issue(repo, branch, bp_orphans, emitter_orphans)
|
||||
except Exception as e:
|
||||
sys.stderr.write(
|
||||
f"::error::failed to file drift issue: {e}\n"
|
||||
)
|
||||
return 1
|
||||
|
||||
if emitter_orphans:
|
||||
print(
|
||||
f"::notice::{len(emitter_orphans)} workflow-emitted context(s) "
|
||||
f"not in BP (informational; Tier 2g handles at PR-time):"
|
||||
)
|
||||
for o in emitter_orphans:
|
||||
print(f"::notice:: - {o}")
|
||||
|
||||
print(
|
||||
f"::notice::BP/emitter match clean: all {len(bp_contexts)} required "
|
||||
f"context(s) have an emitter."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -82,7 +82,7 @@ import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -641,15 +641,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
base_workflows = workflows_at_sha(BASE_SHA)
|
||||
head_workflows = workflows_at_sha(HEAD_SHA)
|
||||
# Ignore workflow files that are identical on both sides — old branches
|
||||
# that haven't rebased onto main carry stale copies of workflows that
|
||||
# were updated later. Comparing those stale copies against the current
|
||||
# base produces false-positive "flips".
|
||||
base_workflows = {
|
||||
p: t for p, t in base_workflows.items()
|
||||
if p in head_workflows and head_workflows[p] != t
|
||||
}
|
||||
head_workflows = {p: t for p, t in head_workflows.items() if p in base_workflows}
|
||||
flips = detect_flips(base_workflows, head_workflows)
|
||||
|
||||
if not flips:
|
||||
|
||||
@@ -1,526 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint_required_context_exists_in_bp — Tier 2g per internal#350.
|
||||
|
||||
Rule
|
||||
----
|
||||
When a PR adds a NEW commit-status emission (a context that didn't
|
||||
exist on the base side), the workflow file must carry one of three
|
||||
directive comments adjacent to the new job:
|
||||
|
||||
(a) `# bp-required: yes`
|
||||
The new context MUST already be in
|
||||
`branch_protections/<branch>.status_check_contexts`. Verified
|
||||
via Gitea API at PR time.
|
||||
|
||||
(b) `# bp-required: pending #NNN`
|
||||
Acknowledged asymmetry; references an OPEN tracking issue that
|
||||
will follow up with the BP PATCH.
|
||||
|
||||
(c) `# bp-exempt: <free-text reason>`
|
||||
Informational job, not intended to be a required gate.
|
||||
|
||||
No directive on a new emitter → FAIL with a 3-option fix-hint.
|
||||
|
||||
The class this prevents
|
||||
-----------------------
|
||||
PR#656 added `CI / all-required (pull_request)` as a sentinel context
|
||||
that workflows emit, but BP did NOT list it. When `platform-build`
|
||||
failed, `all-required` failed, but BP let the PR merge anyway →
|
||||
cascade to mc#664. With this lint, PR#656 would have been blocked
|
||||
until either the BP PATCH ran alongside OR the author added a
|
||||
`bp-required: pending` directive.
|
||||
|
||||
Why directives MUST live in the workflow YAML
|
||||
---------------------------------------------
|
||||
The directive comment lives with the emitter so a scheduled
|
||||
audit (Tier 2f, daily) can read the same source. PR-body-only
|
||||
directives invisibly evaporate on merge — the asymmetry would
|
||||
return to undetected. PR-body claims are advisory; workflow-file
|
||||
comments are the contract.
|
||||
|
||||
How "new emission" is detected
|
||||
------------------------------
|
||||
Diff base..head over `.gitea/workflows/*.yml`. For each YAML file
|
||||
that's added or modified:
|
||||
- Parse both base-side and head-side via PyYAML AST.
|
||||
- Enumerate emitted contexts on each side using the same rules as
|
||||
Tier 2f (workflow.name + job.name|key + event-mapping).
|
||||
- `new_contexts = head_contexts - base_contexts`.
|
||||
|
||||
If `new_contexts` is empty after de-dup, no rule applies → pass.
|
||||
|
||||
Per `feedback_behavior_based_ast_gates`: comment scanning uses raw
|
||||
text in a small window around the job-key line, NOT regex over the
|
||||
full file. This avoids matching `bp-required:` mentioned in a
|
||||
comment unrelated to the new job.
|
||||
|
||||
Exit codes
|
||||
----------
|
||||
0 — no new emissions, all new emissions have valid directives,
|
||||
or BP read errored (graceful-degrade per Tier 2a contract).
|
||||
1 — at least one new emission lacks a directive, or has
|
||||
`bp-required: yes` but the context is missing from BP.
|
||||
2 — env contract violation or YAML parse error.
|
||||
|
||||
Env
|
||||
---
|
||||
BASE_SHA — PR base SHA
|
||||
HEAD_SHA — PR head SHA
|
||||
GITEA_TOKEN — DRIFT_BOT_TOKEN (repo-admin for BP read)
|
||||
GITEA_HOST — e.g. git.moleculesai.app
|
||||
REPO — owner/name
|
||||
BRANCH — defaults to `main`
|
||||
WORKFLOWS_DIR — defaults to `.gitea/workflows`
|
||||
|
||||
Memory cross-links
|
||||
------------------
|
||||
- internal#350 (the RFC that specs this lint)
|
||||
- PR#656 (the empirical case that prompted Tier 2g)
|
||||
- mc#664 (the surfaced cascade)
|
||||
- feedback_phantom_required_check_after_gitea_migration (Tier 2f cousin)
|
||||
- feedback_behavior_based_ast_gates
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
sys.stderr.write(
|
||||
"::error::PyYAML is required. Install with: pip install PyYAML\n"
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# Directive comment patterns. We match `# bp-required:` OR `# bp-exempt:`,
|
||||
# both with optional surrounding whitespace and case-sensitive on the
|
||||
# `bp-` prefix (convention).
|
||||
BP_REQUIRED_YES_RE = re.compile(
|
||||
r"#\s*bp-required:\s*yes\b", re.IGNORECASE
|
||||
)
|
||||
BP_REQUIRED_PENDING_RE = re.compile(
|
||||
r"#\s*bp-required:\s*pending\s*#(?P<num>\d+)\b", re.IGNORECASE
|
||||
)
|
||||
BP_EXEMPT_RE = re.compile(
|
||||
r"#\s*bp-exempt:\s*\S", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
# Gitea event-mapping (same as Tier 2f).
|
||||
_EVENT_MAP = {
|
||||
"pull_request": "pull_request",
|
||||
"pull_request_target": "pull_request",
|
||||
"push": "push",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Env
|
||||
# ---------------------------------------------------------------------------
|
||||
def _env(key: str, default: str | None = None) -> str:
|
||||
v = os.environ.get(key, default)
|
||||
return v if v is not None else ""
|
||||
|
||||
|
||||
def _require_env(key: str) -> str:
|
||||
v = os.environ.get(key)
|
||||
if not v:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API helper (same contract as Tier 2f).
|
||||
# ---------------------------------------------------------------------------
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
) -> tuple[str, Any]:
|
||||
host = _env("GITEA_HOST")
|
||||
token = _env("GITEA_TOKEN")
|
||||
url = f"https://{host}/api/v1{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
if not raw:
|
||||
return ("ok", None)
|
||||
return ("ok", json.loads(raw))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return ("not_found", None)
|
||||
if e.code in (401, 403):
|
||||
return ("forbidden", None)
|
||||
return ("error", None)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return ("error", None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# git helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def git_show(sha: str, path: str) -> str | None:
|
||||
r = subprocess.run(
|
||||
["git", "show", f"{sha}:{path}"], capture_output=True, text=True
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return r.stdout
|
||||
|
||||
|
||||
def git_diff_paths(base: str, head: str) -> list[str]:
|
||||
r = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{base}..{head}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
return [p for p in r.stdout.splitlines() if p.strip()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workflow context enumeration (mirror Tier 2f).
|
||||
# ---------------------------------------------------------------------------
|
||||
def _get_on(d: Any) -> Any:
|
||||
if not isinstance(d, dict):
|
||||
return None
|
||||
if "on" in d:
|
||||
return d["on"]
|
||||
if True in d:
|
||||
return d[True]
|
||||
return None
|
||||
|
||||
|
||||
def _on_events(doc: Any) -> set[str]:
|
||||
on = _get_on(doc)
|
||||
raw: set[str] = set()
|
||||
if on is None:
|
||||
return raw
|
||||
if isinstance(on, str):
|
||||
raw.add(on)
|
||||
elif isinstance(on, list):
|
||||
for e in on:
|
||||
if isinstance(e, str):
|
||||
raw.add(e)
|
||||
elif isinstance(on, dict):
|
||||
for k in on:
|
||||
if isinstance(k, str):
|
||||
raw.add(k)
|
||||
return {_EVENT_MAP[e] for e in raw if e in _EVENT_MAP}
|
||||
|
||||
|
||||
def _job_display(jbody: dict, jkey: str) -> str:
|
||||
n = jbody.get("name") if isinstance(jbody, dict) else None
|
||||
if isinstance(n, str) and n:
|
||||
return n
|
||||
return jkey
|
||||
|
||||
|
||||
def workflow_contexts(doc: Any) -> set[str]:
|
||||
if not isinstance(doc, dict):
|
||||
return set()
|
||||
wf_name = doc.get("name")
|
||||
if not isinstance(wf_name, str) or not wf_name:
|
||||
return set()
|
||||
events = _on_events(doc)
|
||||
if not events:
|
||||
return set()
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return set()
|
||||
out: set[str] = set()
|
||||
for jkey, jbody in jobs.items():
|
||||
if jkey == "__lines__":
|
||||
continue
|
||||
if not isinstance(jbody, dict):
|
||||
continue
|
||||
disp = _job_display(jbody, jkey)
|
||||
for ev in events:
|
||||
out.add(f"{wf_name} / {disp} ({ev})")
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Find the source line of a job-key in a workflow YAML's raw text.
|
||||
# Used to scan for nearby directive comments.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _find_job_key_line(raw_lines: list[str], jkey: str) -> int | None:
|
||||
"""Return 1-based line of `<jkey>:` under jobs:."""
|
||||
in_jobs = False
|
||||
jobs_indent = -1
|
||||
for i, line in enumerate(raw_lines, start=1):
|
||||
stripped = line.lstrip()
|
||||
if stripped.startswith("jobs:"):
|
||||
in_jobs = True
|
||||
jobs_indent = len(line) - len(stripped)
|
||||
continue
|
||||
if in_jobs:
|
||||
# Job key is the next indent level under `jobs:`.
|
||||
indent = len(line) - len(stripped)
|
||||
if stripped and indent <= jobs_indent:
|
||||
# Left the jobs: block
|
||||
in_jobs = False
|
||||
continue
|
||||
if re.match(rf"^\s*{re.escape(jkey)}\s*:", line):
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
_DIRECTIVE_WINDOW = 3 # lines above the job-key line (inclusive)
|
||||
|
||||
|
||||
def find_directive_for_job(
|
||||
raw_text: str, jkey: str
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""Return (kind, value) tuple for the first directive in a small
|
||||
window above the job-key line.
|
||||
|
||||
kind ∈ {"required-yes", "required-pending", "exempt"}.
|
||||
value is the pending-issue number for required-pending, else None.
|
||||
Returns None if no directive found.
|
||||
|
||||
We scan ABOVE the line only (the convention is the directive
|
||||
precedes the job — matches how `# mc#NNN` comments are placed
|
||||
above `continue-on-error: true`). We don't scan inside the job
|
||||
body because steps can produce false positives.
|
||||
"""
|
||||
lines = raw_text.splitlines()
|
||||
line_no = _find_job_key_line(lines, jkey)
|
||||
if line_no is None:
|
||||
return None
|
||||
lo = max(1, line_no - _DIRECTIVE_WINDOW)
|
||||
for i in range(lo, line_no):
|
||||
line = lines[i - 1]
|
||||
m = BP_REQUIRED_PENDING_RE.search(line)
|
||||
if m:
|
||||
return ("required-pending", m.group("num"))
|
||||
if BP_REQUIRED_YES_RE.search(line):
|
||||
return ("required-yes", None)
|
||||
if BP_EXEMPT_RE.search(line):
|
||||
return ("exempt", None)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Map a context back to its emitting (workflow_path, job_key) pair so
|
||||
# we know WHERE to look for the directive comment.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _resolve_emitter(
|
||||
ctx: str, head_workflows: dict[str, tuple[str, Any]]
|
||||
) -> tuple[str, str] | None:
|
||||
"""Return (file_path, job_key) emitting ctx, or None."""
|
||||
m = re.match(r"^(?P<wf>.+?) / (?P<job>.+) \((?P<event>[^)]+)\)$", ctx)
|
||||
if not m:
|
||||
return None
|
||||
target_wf = m.group("wf")
|
||||
target_job_disp = m.group("job")
|
||||
for path, (_raw, doc) in head_workflows.items():
|
||||
if not isinstance(doc, dict):
|
||||
continue
|
||||
if doc.get("name") != target_wf:
|
||||
continue
|
||||
jobs = doc.get("jobs") or {}
|
||||
if not isinstance(jobs, dict):
|
||||
continue
|
||||
for jkey, jbody in jobs.items():
|
||||
if jkey == "__lines__":
|
||||
continue
|
||||
if not isinstance(jbody, dict):
|
||||
continue
|
||||
disp = _job_display(jbody, jkey)
|
||||
if disp == target_job_disp:
|
||||
return (path, jkey)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
def run() -> int:
|
||||
base_sha = _require_env("BASE_SHA")
|
||||
head_sha = _require_env("HEAD_SHA")
|
||||
_require_env("GITEA_TOKEN")
|
||||
_require_env("GITEA_HOST")
|
||||
repo = _require_env("REPO")
|
||||
branch = _env("BRANCH", "main")
|
||||
wf_dir = _env("WORKFLOWS_DIR", ".gitea/workflows")
|
||||
|
||||
# Step 1 — find workflow files changed in the PR.
|
||||
changed = git_diff_paths(base_sha, head_sha)
|
||||
changed_workflows = [
|
||||
p
|
||||
for p in changed
|
||||
if p.startswith(wf_dir + "/")
|
||||
and (p.endswith(".yml") or p.endswith(".yaml"))
|
||||
]
|
||||
if not changed_workflows:
|
||||
print(
|
||||
"::notice::no workflow file changes in this PR; "
|
||||
"lint-required-context-exists-in-bp skipped."
|
||||
)
|
||||
return 0
|
||||
|
||||
# Step 2 — load base+head + compute new contexts.
|
||||
head_workflows: dict[str, tuple[str, Any]] = {}
|
||||
new_contexts: set[str] = set()
|
||||
for path in changed_workflows:
|
||||
base_raw = git_show(base_sha, path)
|
||||
head_raw = git_show(head_sha, path)
|
||||
if head_raw is None:
|
||||
# File deleted on head — no new emission contribution.
|
||||
continue
|
||||
try:
|
||||
head_doc = yaml.safe_load(head_raw)
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(
|
||||
f"::error file={path}::YAML parse error on head: {e}\n"
|
||||
)
|
||||
return 2
|
||||
head_workflows[path] = (head_raw, head_doc)
|
||||
head_ctx = workflow_contexts(head_doc)
|
||||
base_ctx: set[str] = set()
|
||||
if base_raw is not None:
|
||||
try:
|
||||
base_doc = yaml.safe_load(base_raw)
|
||||
except yaml.YAMLError:
|
||||
base_doc = None
|
||||
if base_doc is not None:
|
||||
base_ctx = workflow_contexts(base_doc)
|
||||
new_contexts |= (head_ctx - base_ctx)
|
||||
|
||||
if not new_contexts:
|
||||
print(
|
||||
"::notice::no new context emissions detected in this PR; "
|
||||
"lint-required-context-exists-in-bp skipped."
|
||||
)
|
||||
return 0
|
||||
|
||||
# Step 3 — fetch BP context list.
|
||||
status, bp = api("GET", f"/repos/{repo}/branch_protections/{branch}")
|
||||
bp_contexts: set[str] = set()
|
||||
if status == "forbidden":
|
||||
sys.stderr.write(
|
||||
f"::error::GET branch_protections/{branch} returned HTTP 403 — "
|
||||
f"DRIFT_BOT_TOKEN lacks repo-admin scope. Cannot verify "
|
||||
f"bp-required directives; skipping lint with exit 0 per "
|
||||
f"Tier 2a contract. Fix the token, not the lint.\n"
|
||||
)
|
||||
return 0
|
||||
elif status == "not_found":
|
||||
# Branch has no protection — nothing to verify against; the
|
||||
# bp-required: yes directive can't be satisfied. Treat as
|
||||
# graceful-skip rather than red-X.
|
||||
print(
|
||||
f"::notice::branch '{branch}' has no protection; cannot verify "
|
||||
f"bp-required directives. Skipping (exit 0)."
|
||||
)
|
||||
return 0
|
||||
elif status == "ok" and isinstance(bp, dict):
|
||||
bp_contexts = set(bp.get("status_check_contexts") or [])
|
||||
else:
|
||||
sys.stderr.write(
|
||||
f"::error::branch_protections/{branch} response unexpected; "
|
||||
f"status={status}. Treating as transient; exit 0.\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Step 4 — validate each new emission's directive.
|
||||
violations: list[str] = []
|
||||
for ctx in sorted(new_contexts):
|
||||
emitter = _resolve_emitter(ctx, head_workflows)
|
||||
if emitter is None:
|
||||
# Shouldn't happen — we just derived ctx from head_workflows.
|
||||
# Belt-and-suspenders fallback.
|
||||
violations.append(
|
||||
f"::error::new emission '{ctx}' (could not resolve emitter "
|
||||
f"file/job — bug in lint?)"
|
||||
)
|
||||
continue
|
||||
file_path, jkey = emitter
|
||||
raw_text, _ = head_workflows[file_path]
|
||||
directive = find_directive_for_job(raw_text, jkey)
|
||||
if directive is None:
|
||||
violations.append(
|
||||
f"::error file={file_path}::lint-required-context-exists-in-bp "
|
||||
f"(Tier 2g): NEW emission `{ctx}` (job '{jkey}') has no "
|
||||
f"directive comment. Add ONE of these comments on the line "
|
||||
f"directly above `{jkey}:` (within {_DIRECTIVE_WINDOW} lines):\n"
|
||||
f" - `# bp-required: yes` — and ensure the context is "
|
||||
f"already in branch_protections/{branch}.status_check_contexts.\n"
|
||||
f" - `# bp-required: pending #NNN` — acknowledged asymmetry, "
|
||||
f"references the tracking issue for the BP PATCH.\n"
|
||||
f" - `# bp-exempt: <reason>` — informational job, not a gate.\n"
|
||||
f"Memory: internal#350 (PR#656 + mc#664 empirical case)."
|
||||
)
|
||||
continue
|
||||
kind, value = directive
|
||||
if kind == "exempt":
|
||||
print(f"::notice::{ctx}: bp-exempt directive present, OK.")
|
||||
continue
|
||||
if kind == "required-pending":
|
||||
print(
|
||||
f"::notice::{ctx}: bp-required: pending #{value} — "
|
||||
f"acknowledged asymmetry, OK."
|
||||
)
|
||||
continue
|
||||
if kind == "required-yes":
|
||||
if ctx in bp_contexts:
|
||||
print(
|
||||
f"::notice::{ctx}: bp-required: yes, and context is in "
|
||||
f"BP, OK."
|
||||
)
|
||||
else:
|
||||
violations.append(
|
||||
f"::error file={file_path}::lint-required-context-exists-in-bp "
|
||||
f"(Tier 2g): job '{jkey}' has `bp-required: yes` "
|
||||
f"directive but its emitted context `{ctx}` is NOT in "
|
||||
f"`branch_protections/{branch}.status_check_contexts`. "
|
||||
f"FIX: either (a) add `{ctx}` to BP (Owners-tier PATCH), "
|
||||
f"or (b) downgrade the directive to "
|
||||
f"`# bp-required: pending #NNN` referencing the tracker "
|
||||
f"for the pending BP PATCH."
|
||||
)
|
||||
|
||||
if violations:
|
||||
print(
|
||||
f"::error::lint-required-context-exists-in-bp: "
|
||||
f"{len(violations)} violation(s) across "
|
||||
f"{len(changed_workflows)} changed workflow file(s)."
|
||||
)
|
||||
for v in violations:
|
||||
print(v)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"::notice::lint-required-context-exists-in-bp: "
|
||||
f"{len(new_contexts)} new emission(s) all directive-validated."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -61,7 +61,6 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -90,28 +89,6 @@ API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
# match by exact title without parsing.
|
||||
TITLE_PREFIX = "[main-red]"
|
||||
|
||||
# Contexts that are scheduled or non-required — their pending/failure
|
||||
# state should not block stale-issue closeout (mc#1789).
|
||||
SCHEDULED_CONTEXT_PATTERNS = (
|
||||
"Staging SaaS smoke",
|
||||
"Continuous synthetic E2E",
|
||||
"main-red-watchdog",
|
||||
"ci-arm64-advisory",
|
||||
)
|
||||
|
||||
# Settling window (seconds) between initial red detection and the
|
||||
# pre-file recheck. The recheck filters out the two largest false-
|
||||
# positive classes seen in mc#1597..1630 (task #394, 2026-05-21):
|
||||
# 1. HEAD moved on (a new commit landed mid-tick) — the prior red SHA
|
||||
# is no longer authoritative; let the next cron tick re-evaluate.
|
||||
# 2. Combined status recovered on the SAME SHA (transient
|
||||
# cancel-cascade rolled forward to success on retry).
|
||||
# 90s is well below the hourly cron cadence; a real failure that
|
||||
# persists past it is the one we want surfaced.
|
||||
# Override with WATCHDOG_RECHECK_DELAY_SECS for tests / local probes
|
||||
# (the test suite stubs time.sleep to a no-op).
|
||||
RECHECK_DELAY_SECS = int(_env("WATCHDOG_RECHECK_DELAY_SECS", default="90"))
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only.
|
||||
@@ -195,49 +172,6 @@ def api(
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# action_run.status resolver — extensibility hook for task #394.
|
||||
# --------------------------------------------------------------------------
|
||||
def _resolve_action_run_status(target_url: str) -> int | None:
|
||||
"""Resolve the underlying Gitea `action_run.status` integer for the
|
||||
run referenced by `target_url`, returning None if the resolver
|
||||
cannot reach an authoritative source from the runner.
|
||||
|
||||
Canonical Gitea 1.22.6 enum (per `models/actions/status.go` +
|
||||
`reference_gitea_action_status_enum_corrected_2026_05_19`):
|
||||
1=Success, 2=Failure, 3=Cancelled, 4=Skipped,
|
||||
5=Waiting, 6=Running, 7=Blocked
|
||||
Only `status == 2` is a real defect; status=3 is cancel-cascade and
|
||||
status=1 is an emission artifact (Gitea wrote a 'failure' commit_status
|
||||
row for a run that actually succeeded — observed empirically on
|
||||
`publish-canvas-image` jobs at SHAs in mc#1597..1630).
|
||||
|
||||
CURRENT STATE (2026-05-20, verified): Gitea 1.22.6 exposes NO REST
|
||||
endpoint for `action_run.status`. Probed:
|
||||
/api/v1/repos/{o}/{r}/actions/runs/{id} → HTTP 404
|
||||
/api/v1/repos/{o}/{r}/actions/jobs/{id} → HTTP 404
|
||||
/api/v1/repos/{o}/{r}/actions/tasks/{id} → HTTP 404
|
||||
/swagger.v1.json paths containing 'actions' → secrets+variables+runners only
|
||||
The SPA backend (`/{repo}/actions/runs/{id}/jobs/{idx}` POST) requires
|
||||
a session CSRF token, unreachable from a runner. The only authoritative
|
||||
source today is direct DB access (`mol_action_status` on op-host,
|
||||
`docker exec molecule-postgres-1 psql ...`), which the runner cannot
|
||||
reach.
|
||||
|
||||
Therefore: this hook returns None on every call. Callers MUST fall
|
||||
back to the description-string filter (existing) plus the HEAD
|
||||
recheck (this PR). When a future Gitea release (>=1.23 expected) or
|
||||
an op-host proxy exposes the endpoint, replace the body of this
|
||||
function with an `api(...)` call — the caller contract is stable.
|
||||
|
||||
See also:
|
||||
- `reference_chronic_red_sweep_cancelled_vs_failed_filter`
|
||||
- `feedback_gitea_status_enum_use_helper_not_raw_int`
|
||||
"""
|
||||
_ = target_url # noqa: F841 — intentional placeholder
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gitea reads
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -274,11 +208,6 @@ def get_combined_status(sha: str) -> dict:
|
||||
return body
|
||||
|
||||
|
||||
def _entry_state(s: dict) -> str:
|
||||
"""Per-entry status key in Gitea 1.22.6 is `status`; fall back to `state`."""
|
||||
return s.get("status") or s.get("state") or ""
|
||||
|
||||
|
||||
def is_red(status: dict) -> tuple[bool, list[dict]]:
|
||||
"""Return (is_red, failed_statuses).
|
||||
|
||||
@@ -289,31 +218,6 @@ def is_red(status: dict) -> tuple[bool, list[dict]]:
|
||||
|
||||
`failed_statuses` is the list of per-context entries whose own
|
||||
`state` is in the red set; useful for the issue body.
|
||||
|
||||
Cancel-cascade filter (mc#1564, 2026-05-19):
|
||||
Gitea maps BOTH `action_run.status=2 (Failure)` AND
|
||||
`action_run.status=3 (Cancelled)` to commit-status string
|
||||
`"failure"`. On a busy main with
|
||||
`concurrency: cancel-in-progress: true`, every merge burst
|
||||
cancels prior in-flight runs (status=3) — those bubble to the
|
||||
combined-status `failure` and inflate the watchdog's red%,
|
||||
generating phantom `[main-red]` issues (mc#1562/#1552/#1540/...).
|
||||
Canonical Gitea 1.22.6 enum per `models/actions/status.go` +
|
||||
`reference_gitea_action_status_enum_corrected_2026_05_19`:
|
||||
1=Success, 2=Failure, 3=Cancelled, 4=Skipped,
|
||||
5=Waiting, 6=Running, 7=Blocked
|
||||
We only want status=2 (real defects) to file. At the
|
||||
commit-status layer we don't have the integer enum directly
|
||||
(only the `failure` rollup string), so we use the description
|
||||
string Gitea writes when a run is cancelled — empirically
|
||||
`"Has been cancelled"` (verified 2026-05-19 via #1562 body).
|
||||
Real failures show `"Failing after Ns"` and are unaffected.
|
||||
This is option B from mc#1564 (description-string filter, no
|
||||
extra API call). Description-string stability is a soft contract
|
||||
with Gitea; if a future release renames it, the cancel-cascade
|
||||
entries will simply leak back through (visible-not-silent), and
|
||||
we'll either re-pin the string or upgrade to option A (resolve
|
||||
the underlying action_run.status integer via target_url).
|
||||
"""
|
||||
combined = status.get("state")
|
||||
statuses = status.get("statuses") or []
|
||||
@@ -326,30 +230,14 @@ def is_red(status: dict) -> tuple[bool, list[dict]]:
|
||||
# "no per-context entries were in a red state" fallback even when
|
||||
# the combined-state correctly flagged red. See
|
||||
# `feedback_smoke_test_vendor_truth_not_shape_match`.
|
||||
def _is_cancel_cascade(s: dict) -> bool:
|
||||
"""status=3 entry per Gitea 1.22.6 description-string contract.
|
||||
Match exactly (after strip) — substring match would catch
|
||||
legitimate test names like "Has been cancelled by the user
|
||||
unexpectedly" in failure logs."""
|
||||
desc = (s.get("description") or "").strip()
|
||||
return desc == "Has been cancelled"
|
||||
def _entry_state(s: dict) -> str:
|
||||
return s.get("status") or s.get("state") or ""
|
||||
|
||||
failed = [
|
||||
s for s in statuses
|
||||
if isinstance(s, dict)
|
||||
and _entry_state(s) in red_states
|
||||
and not _is_cancel_cascade(s)
|
||||
if isinstance(s, dict) and _entry_state(s) in red_states
|
||||
]
|
||||
# Combined state alone is no longer sufficient — combined=failure
|
||||
# may be 100% cancel-cascade. Drive `red` off the FILTERED list:
|
||||
# if every red-shaped per-entry was cancel-cascade, `failed` is
|
||||
# empty and we report green. Combined-failure with no per-entry
|
||||
# detail (empty `statuses[]`) still trips red — that's the
|
||||
# "CI emitter set combined-status directly" edge case from
|
||||
# render_body's fallback path; we keep filing on it so the
|
||||
# operator sees the breadcrumb.
|
||||
combined_red_no_detail = combined in red_states and not statuses
|
||||
return (bool(failed) or combined_red_no_detail, failed)
|
||||
return (combined in red_states or bool(failed), failed)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -364,15 +252,6 @@ def title_for(sha: str) -> str:
|
||||
return f"{TITLE_PREFIX} {REPO}: {sha[:10]}"
|
||||
|
||||
|
||||
def _is_scheduled_context(context: str) -> bool:
|
||||
"""Return True if `context` is a known scheduled/non-required job.
|
||||
|
||||
These contexts run on a schedule and should not block stale-issue
|
||||
closeout when main's required CI has recovered (mc#1789).
|
||||
"""
|
||||
return any(pattern.lower() in context.lower() for pattern in SCHEDULED_CONTEXT_PATTERNS)
|
||||
|
||||
|
||||
def list_open_red_issues() -> list[dict]:
|
||||
"""All open issues whose title starts with `[main-red] {repo}: `.
|
||||
|
||||
@@ -382,34 +261,23 @@ def list_open_red_issues() -> list[dict]:
|
||||
file-or-update path to POST a duplicate — exactly the regression
|
||||
class the helper-raises contract closes.
|
||||
|
||||
Pagination is exhausted (mc#1789). The old "by design ≤ 1" invariant
|
||||
was false — backlog can exceed 50 open issues.
|
||||
Gitea issue search returns at most 50/page; we only need open
|
||||
`[main-red]` issues which are by design ≤ 1 at any time per repo,
|
||||
so a single page is enough.
|
||||
"""
|
||||
prefix = f"{TITLE_PREFIX} {REPO}: "
|
||||
all_issues: list[dict] = []
|
||||
page = 1
|
||||
limit = 50
|
||||
while True:
|
||||
_, results = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={"state": "open", "type": "issues", "limit": str(limit), "page": str(page)},
|
||||
_, results = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={"state": "open", "type": "issues", "limit": "50"},
|
||||
)
|
||||
if not isinstance(results, list):
|
||||
raise ApiError(
|
||||
f"issue search returned non-list body (got {type(results).__name__})"
|
||||
)
|
||||
if not isinstance(results, list):
|
||||
raise ApiError(
|
||||
f"issue search returned non-list body (got {type(results).__name__})"
|
||||
)
|
||||
matched = [
|
||||
i for i in results
|
||||
if isinstance(i, dict)
|
||||
prefix = f"{TITLE_PREFIX} {REPO}: "
|
||||
return [i for i in results if isinstance(i, dict)
|
||||
and isinstance(i.get("title"), str)
|
||||
and i["title"].startswith(prefix)
|
||||
]
|
||||
all_issues.extend(matched)
|
||||
if len(results) < limit:
|
||||
break
|
||||
page += 1
|
||||
return all_issues
|
||||
and i["title"].startswith(prefix)]
|
||||
|
||||
|
||||
def find_open_issue_for_sha(sha: str) -> dict | None:
|
||||
@@ -605,156 +473,10 @@ def file_or_update_red(
|
||||
sys.stderr.write(f"::warning::label '{RED_LABEL}' not found on repo\n")
|
||||
|
||||
|
||||
def close_stale_red_issues(
|
||||
current_sha: str,
|
||||
current_status: dict,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> int:
|
||||
"""Close open [main-red] issues whose specific failing contexts have
|
||||
all recovered on `current_sha`, even though `main` is still red for
|
||||
other reasons (mc#1789).
|
||||
|
||||
When main stays red across consecutive SHAs for *different* causes,
|
||||
`close_open_red_issues_for_other_shas` never fires (it only runs when
|
||||
main is green). This function prevents stale issues from accumulating
|
||||
indefinitely by comparing per-context recovery across SHAs.
|
||||
|
||||
An issue is considered stale when every context that was in a failed
|
||||
state on the issue's SHA is now either `success` on the current HEAD
|
||||
or absent (workflow removed / renamed). Issues whose original SHA had
|
||||
a combined-red-with-no-detail (empty statuses list) are skipped — we
|
||||
cannot verify recovery without per-context data.
|
||||
|
||||
Returns the number of issues closed.
|
||||
"""
|
||||
open_red = list_open_red_issues()
|
||||
if not open_red:
|
||||
return 0
|
||||
|
||||
current_statuses = current_status.get("statuses") or []
|
||||
closed = 0
|
||||
|
||||
for issue in open_red:
|
||||
title = issue.get("title", "")
|
||||
prefix = f"{TITLE_PREFIX} {REPO}: "
|
||||
if not title.startswith(prefix):
|
||||
continue
|
||||
short_sha = title[len(prefix):]
|
||||
if short_sha == current_sha[:10]:
|
||||
continue
|
||||
|
||||
# Query status for the old SHA. Short SHA should resolve; if it
|
||||
# doesn't (GC'd, force-pushed, ambiguous), skip conservatively.
|
||||
try:
|
||||
old_status = get_combined_status(short_sha)
|
||||
except ApiError:
|
||||
continue
|
||||
|
||||
old_red, old_failed = is_red(old_status)
|
||||
if not old_red:
|
||||
# Open issue for a now-green SHA — close it via the normal path.
|
||||
num = issue.get("number")
|
||||
if isinstance(num, int):
|
||||
comment = (
|
||||
f"Commit `{short_sha}` is no longer red. Closing as the "
|
||||
f"failure context has recovered or expired."
|
||||
)
|
||||
if dry_run:
|
||||
print(
|
||||
f"::notice::[dry-run] would close issue #{num} "
|
||||
f"({title}) — old SHA is now green"
|
||||
)
|
||||
closed += 1
|
||||
continue
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}/comments",
|
||||
body={"body": comment},
|
||||
)
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}",
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print(
|
||||
f"::notice::Closed stale main-red issue #{num} "
|
||||
f"(old SHA {short_sha} is now green)"
|
||||
)
|
||||
closed += 1
|
||||
continue
|
||||
|
||||
if not old_failed:
|
||||
# Combined red with no per-context detail — can't verify recovery.
|
||||
continue
|
||||
|
||||
# Verify every failed context from the old SHA has recovered.
|
||||
all_recovered = True
|
||||
recovered_ctxs: list[str] = []
|
||||
still_failing_ctxs: list[str] = []
|
||||
for s in old_failed:
|
||||
ctx = s.get("context", "")
|
||||
if not ctx:
|
||||
continue
|
||||
current_match = None
|
||||
for cs in current_statuses:
|
||||
if isinstance(cs, dict) and cs.get("context") == ctx:
|
||||
current_match = cs
|
||||
break
|
||||
if current_match is None:
|
||||
recovered_ctxs.append(ctx)
|
||||
elif _entry_state(current_match) == "success":
|
||||
recovered_ctxs.append(ctx)
|
||||
else:
|
||||
all_recovered = False
|
||||
still_failing_ctxs.append(ctx)
|
||||
|
||||
if not all_recovered:
|
||||
continue
|
||||
|
||||
num = issue.get("number")
|
||||
if not isinstance(num, int):
|
||||
continue
|
||||
|
||||
comment = (
|
||||
f"The failing contexts from this SHA (`{short_sha}`) have "
|
||||
f"recovered on current HEAD `{current_sha[:10]}`: "
|
||||
f"{', '.join(recovered_ctxs)}. "
|
||||
f"Main is still red for other reasons; see the current "
|
||||
f"`[main-red]` issue for `{current_sha[:10]}`."
|
||||
)
|
||||
if dry_run:
|
||||
print(
|
||||
f"::notice::[dry-run] would close stale issue #{num} "
|
||||
f"({title}) — contexts recovered"
|
||||
)
|
||||
closed += 1
|
||||
continue
|
||||
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}/comments",
|
||||
body={"body": comment},
|
||||
)
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}",
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print(
|
||||
f"::notice::Closed stale main-red issue #{num} "
|
||||
f"(contexts recovered at {current_sha[:10]})"
|
||||
)
|
||||
closed += 1
|
||||
|
||||
return closed
|
||||
|
||||
|
||||
def close_open_red_issues_for_other_shas(
|
||||
current_sha: str,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
close_same_sha: bool = False,
|
||||
) -> int:
|
||||
"""When main is green at current_sha, close any open `[main-red]`
|
||||
issues whose title references a different SHA. Returns the number
|
||||
@@ -763,25 +485,15 @@ def close_open_red_issues_for_other_shas(
|
||||
Lineage note: we only close issues whose title prefix matches; if
|
||||
a human renamed the issue or added a suffix this won't touch it.
|
||||
That's intentional — manual editorial state takes precedence.
|
||||
|
||||
Args:
|
||||
close_same_sha: set True when the caller already knows main is
|
||||
green at current_sha (e.g. recovery block) and wants to close
|
||||
the open issue for THIS SHA too. Defaults False so the
|
||||
green-path callers never accidentally close an issue they just
|
||||
filed on the same tick.
|
||||
"""
|
||||
target_title = title_for(current_sha)
|
||||
open_red = list_open_red_issues()
|
||||
closed = 0
|
||||
for issue in open_red:
|
||||
if issue.get("title") == target_title:
|
||||
if not close_same_sha:
|
||||
# Same SHA — caller should not have invoked this if main is
|
||||
# green. Skip defensively (guards against green-path callers
|
||||
# that accidentally pass the SHA they just filed for).
|
||||
continue
|
||||
# close_same_sha=True: close even this SHA's issue (recovery path)
|
||||
# Same SHA — caller should not have invoked this if main is
|
||||
# green. Skip defensively.
|
||||
continue
|
||||
num = issue.get("number")
|
||||
if not isinstance(num, int):
|
||||
continue
|
||||
@@ -858,130 +570,29 @@ def run_once(*, dry_run: bool = False) -> int:
|
||||
}
|
||||
|
||||
if red:
|
||||
# HEAD recheck (task #394 — guards mc#1597..1630 false-positive
|
||||
# cluster). After the initial detection, wait RECHECK_DELAY_SECS
|
||||
# (default 90s; tests stub time.sleep) and re-evaluate:
|
||||
#
|
||||
# 1. Re-fetch HEAD SHA. If HEAD moved, a new commit landed
|
||||
# mid-tick — the prior red SHA is no longer authoritative
|
||||
# and the next cron run will re-evaluate against the new
|
||||
# HEAD. Skip-file.
|
||||
#
|
||||
# 2. If HEAD unchanged, re-fetch the combined status. If it
|
||||
# recovered (combined state no longer in {failure,error}
|
||||
# after the cancel-cascade filter), a transient retry
|
||||
# rolled the run forward. Skip-file.
|
||||
#
|
||||
# Both paths emit a Loki event distinguishable from the real
|
||||
# `main_red_detected` so obs queries can track filter activity.
|
||||
# The settling window is well below the hourly cron cadence —
|
||||
# genuine failures persist past it and are surfaced normally.
|
||||
time.sleep(RECHECK_DELAY_SECS)
|
||||
|
||||
recheck_sha = get_head_sha(WATCH_BRANCH)
|
||||
if recheck_sha != sha:
|
||||
emit_loki_event("main_red_skipped_head_drift", sha, [])
|
||||
print(
|
||||
f"::notice::skip-file (HEAD moved): initial red at "
|
||||
f"{sha[:10]} but HEAD is now {recheck_sha[:10]} on "
|
||||
f"{WATCH_BRANCH}; next cron tick will re-evaluate."
|
||||
)
|
||||
# HEAD drifted — close any stale main-red issue for the prior SHA
|
||||
# before returning, so we don't leave stale open issues when main
|
||||
# is no longer pointing at the red commit.
|
||||
close_open_red_issues_for_other_shas(recheck_sha, dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
recheck_status = get_combined_status(sha)
|
||||
recheck_red, recheck_failed = is_red(recheck_status)
|
||||
if not recheck_red:
|
||||
emit_loki_event("main_red_skipped_recovered", sha, [])
|
||||
print(
|
||||
f"::notice::skip-file (recovered after settling): "
|
||||
f"combined state at {sha[:10]} flipped to "
|
||||
f"{recheck_status.get('state')!r} on recheck; "
|
||||
f"initial red was a transient cancel-cascade."
|
||||
)
|
||||
# CI recovered on the same SHA — close any stale main-red issue
|
||||
# that was filed on a prior tick for this SHA.
|
||||
close_open_red_issues_for_other_shas(sha, dry_run=dry_run, close_same_sha=True)
|
||||
return 0
|
||||
|
||||
# Still red after settling — file/update. Use the recheck data
|
||||
# as authoritative so the issue body reflects the latest state.
|
||||
failed = recheck_failed
|
||||
debug["recheck_combined_state"] = recheck_status.get("state")
|
||||
debug["recheck_failed_contexts"] = [
|
||||
s.get("context") for s in failed
|
||||
]
|
||||
|
||||
failed_ctxs = [s.get("context") for s in failed if s.get("context")]
|
||||
emit_loki_event("main_red_detected", sha, failed_ctxs)
|
||||
print(f"::warning::main is RED at {sha[:10]} on {WATCH_BRANCH}: "
|
||||
f"{len(failed)} failed context(s)")
|
||||
file_or_update_red(sha, failed, debug, dry_run=dry_run)
|
||||
stale_closed = close_stale_red_issues(sha, recheck_status, dry_run=dry_run)
|
||||
if stale_closed:
|
||||
emit_loki_event("main_red_stale_closed", sha, [])
|
||||
print(
|
||||
f"::notice::Closed {stale_closed} stale main-red issue(s) "
|
||||
f"whose contexts recovered at {sha[:10]}"
|
||||
)
|
||||
else:
|
||||
# Green or pending-with-no-real-failures. Close stale issues
|
||||
# from earlier SHAs when required CI has recovered.
|
||||
#
|
||||
# mc#1789: main often sits at combined `pending` because
|
||||
# scheduled/non-required contexts (Staging SaaS smoke,
|
||||
# Continuous synthetic E2E, main-red-watchdog itself,
|
||||
# ci-arm64-advisory) are still running. We close stale issues
|
||||
# as long as no *non-scheduled* context has failed and no
|
||||
# *non-scheduled* context is still pending — i.e. required CI
|
||||
# is effectively green.
|
||||
#
|
||||
# The success-only gate is preserved for the canonical green
|
||||
# path; the extended check below only fires when combined is
|
||||
# `pending` but all required work is done.
|
||||
combined_state = status.get("state")
|
||||
if combined_state == "success":
|
||||
should_close = True
|
||||
close_reason = "GREEN"
|
||||
else:
|
||||
statuses = status.get("statuses") or []
|
||||
non_scheduled_pending = [
|
||||
s for s in statuses
|
||||
if isinstance(s, dict)
|
||||
and (_entry_state(s) == "pending")
|
||||
and not _is_scheduled_context(s.get("context", ""))
|
||||
]
|
||||
non_scheduled_failed = [
|
||||
s for s in statuses
|
||||
if isinstance(s, dict)
|
||||
and (_entry_state(s) in {"failure", "error"})
|
||||
and not _is_scheduled_context(s.get("context", ""))
|
||||
]
|
||||
# Cancel-cascade already filtered by is_red(); red=False
|
||||
# here means no real failures. We additionally check that
|
||||
# no non-scheduled context is still pending.
|
||||
should_close = not non_scheduled_pending and not non_scheduled_failed
|
||||
close_reason = "pending-but-required-green"
|
||||
|
||||
if should_close:
|
||||
# Green (or pending — pending is treated as not-red so we don't
|
||||
# spam during the post-merge CI window). Close any stale issues
|
||||
# from earlier SHAs only when we're actually green; pending
|
||||
# means CI hasn't finished and the prior issue might still be
|
||||
# accurate.
|
||||
if status.get("state") == "success":
|
||||
closed = close_open_red_issues_for_other_shas(sha, dry_run=dry_run)
|
||||
if closed:
|
||||
emit_loki_event(
|
||||
"main_returned_to_green", sha,
|
||||
[],
|
||||
)
|
||||
print(
|
||||
f"::notice::main is {close_reason} at {sha[:10]} on {WATCH_BRANCH} "
|
||||
f"(closed {closed} stale issue(s))"
|
||||
)
|
||||
print(f"::notice::main is GREEN at {sha[:10]} on {WATCH_BRANCH} "
|
||||
f"(closed {closed} stale issue(s))")
|
||||
else:
|
||||
print(
|
||||
f"::notice::main has pending-or-failed required CI at {sha[:10]} "
|
||||
f"on {WATCH_BRANCH} (combined state={combined_state!r}; no action)"
|
||||
)
|
||||
print(f"::notice::main is PENDING at {sha[:10]} on {WATCH_BRANCH} "
|
||||
f"(combined state={status.get('state')!r}; no action)")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -1,470 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Production auto-deploy helpers for Gitea Actions.
|
||||
|
||||
The workflow keeps network side effects in shell/curl, but centralizes the
|
||||
release decision shape here so it has unit coverage: disable flag parsing,
|
||||
target tag selection, CP payload construction, and status-context selection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from urllib.parse import quote
|
||||
|
||||
TRUE_VALUES = {"1", "true", "yes", "on", "disabled", "disable"}
|
||||
PROD_CP_URL = "https://api.moleculesai.app"
|
||||
DEFAULT_REQUIRED_CONTEXTS = [
|
||||
"CI / all-required (push)",
|
||||
"Secret scan / Scan diff for credential-shaped strings (push)",
|
||||
]
|
||||
TERMINAL_FAILURE_STATES = {"failure", "error", "cancelled", "canceled", "skipped"}
|
||||
REDEPLOY_PATH = "/cp/admin/tenants/redeploy-fleet"
|
||||
|
||||
|
||||
def truthy_flag(value: str | None) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
return value.strip().lower() in TRUE_VALUES
|
||||
|
||||
|
||||
def _int_env(env: dict[str, str], name: str, default: int, minimum: int = 1) -> int:
|
||||
raw = env.get(name, "")
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{name} must be an integer, got {raw!r}") from exc
|
||||
if value < minimum:
|
||||
raise ValueError(f"{name} must be >= {minimum}, got {value}")
|
||||
return value
|
||||
|
||||
|
||||
def build_plan(env: dict[str, str]) -> dict:
|
||||
sha = env.get("GITHUB_SHA", "").strip()
|
||||
if not sha:
|
||||
raise ValueError("GITHUB_SHA is required")
|
||||
|
||||
disabled_value = env.get("PROD_AUTO_DEPLOY_DISABLED", "")
|
||||
if truthy_flag(disabled_value):
|
||||
return {
|
||||
"enabled": False,
|
||||
"sha": sha,
|
||||
"disabled_reason": f"PROD_AUTO_DEPLOY_DISABLED={disabled_value}",
|
||||
}
|
||||
|
||||
short_sha = sha[:7]
|
||||
target_tag = env.get("PROD_AUTO_DEPLOY_TARGET_TAG", "").strip() or f"staging-{short_sha}"
|
||||
canary_slug = env.get("PROD_AUTO_DEPLOY_CANARY_SLUG", "hongming").strip()
|
||||
body = {
|
||||
"target_tag": target_tag,
|
||||
"soak_seconds": _int_env(env, "PROD_AUTO_DEPLOY_SOAK_SECONDS", 60, minimum=0),
|
||||
"batch_size": _int_env(env, "PROD_AUTO_DEPLOY_BATCH_SIZE", 3),
|
||||
"dry_run": truthy_flag(env.get("PROD_AUTO_DEPLOY_DRY_RUN", "")),
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. This caller is the
|
||||
# production auto-deploy step that rolls every live tenant (canary
|
||||
# + fan-out), no slug scoping, so confirm:true is correct.
|
||||
"confirm": True,
|
||||
}
|
||||
if canary_slug:
|
||||
body["canary_slug"] = canary_slug
|
||||
|
||||
cp_url = env.get("CP_URL", "").strip() or PROD_CP_URL
|
||||
if cp_url != PROD_CP_URL and not truthy_flag(env.get("PROD_ALLOW_NON_PROD_CP_URL", "")):
|
||||
raise ValueError(
|
||||
f"Refusing production deploy to CP_URL={cp_url!r}; "
|
||||
f"set PROD_ALLOW_NON_PROD_CP_URL=true for an explicit non-prod drill"
|
||||
)
|
||||
|
||||
return {
|
||||
"enabled": True,
|
||||
"sha": sha,
|
||||
"short_sha": short_sha,
|
||||
"target_tag": target_tag,
|
||||
"cp_url": cp_url,
|
||||
"body": body,
|
||||
}
|
||||
|
||||
|
||||
def latest_status_for_context(statuses: list[dict], context: str) -> dict | None:
|
||||
"""Return the first matching status.
|
||||
|
||||
Gitea's combined-status response is newest-first in practice. The merge
|
||||
queue relies on the same contract; keeping the selector explicit makes
|
||||
stale duplicate contexts easy to test.
|
||||
"""
|
||||
|
||||
for status in statuses:
|
||||
if status.get("context") == context:
|
||||
return status
|
||||
return None
|
||||
|
||||
|
||||
def ci_context_state(statuses: list[dict], context: str) -> str:
|
||||
status = latest_status_for_context(statuses, context)
|
||||
if not status:
|
||||
return "missing"
|
||||
return str(status.get("status") or status.get("state") or "missing").lower()
|
||||
|
||||
|
||||
def context_is_satisfied(state: str) -> bool:
|
||||
return state == "success"
|
||||
|
||||
|
||||
def context_is_terminal_failure(state: str) -> bool:
|
||||
return state in TERMINAL_FAILURE_STATES
|
||||
|
||||
|
||||
def required_contexts(env: dict[str, str]) -> list[str]:
|
||||
raw = env.get("PROD_AUTO_DEPLOY_REQUIRED_CONTEXTS", "")
|
||||
if not raw.strip():
|
||||
return DEFAULT_REQUIRED_CONTEXTS
|
||||
return [line.strip() for line in raw.replace(",", "\n").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def chunks(items: list[str], size: int) -> list[list[str]]:
|
||||
return [items[i : i + size] for i in range(0, len(items), size)]
|
||||
|
||||
|
||||
class RolloutFailed(RuntimeError):
|
||||
def __init__(self, message: str, response: dict):
|
||||
super().__init__(message)
|
||||
self.response = response
|
||||
|
||||
|
||||
def slugs_from_redeploy_response(body: dict) -> list[str]:
|
||||
slugs: list[str] = []
|
||||
for row in body.get("results") or []:
|
||||
slug = str(row.get("slug") or "").strip()
|
||||
if slug:
|
||||
slugs.append(slug)
|
||||
return slugs
|
||||
|
||||
|
||||
def scoped_redeploy_body(base: dict, slugs: list[str]) -> dict:
|
||||
body = dict(base)
|
||||
body.pop("canary_slug", None)
|
||||
body["only_slugs"] = slugs
|
||||
body["soak_seconds"] = 0
|
||||
body["batch_size"] = max(1, len(slugs))
|
||||
return body
|
||||
|
||||
|
||||
def cp_api_json(method: str, url: str, token: str, body: dict | None = None) -> tuple[int, dict]:
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
return resp.status, json.loads(resp.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
parsed = {"error": raw[:500]}
|
||||
return exc.code, parsed
|
||||
|
||||
|
||||
def plan_rollout_slugs(cp_url: str, token: str, body: dict, redeploy=None) -> list[str]:
|
||||
if redeploy is None:
|
||||
redeploy = redeploy_scoped
|
||||
dry_run_body = dict(body)
|
||||
dry_run_body["dry_run"] = True
|
||||
status, resp = redeploy(cp_url, token, dry_run_body)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"dry-run redeploy-fleet returned HTTP {status}: {resp.get('error', '')}")
|
||||
if resp.get("ok") is not True:
|
||||
raise RuntimeError(f"dry-run redeploy-fleet reported ok={resp.get('ok')}: {resp.get('error', '')}")
|
||||
slugs = slugs_from_redeploy_response(resp)
|
||||
if not slugs:
|
||||
raise RuntimeError("dry-run redeploy-fleet returned no rollout candidates")
|
||||
return slugs
|
||||
|
||||
|
||||
def redeploy_scoped(cp_url: str, token: str, body: dict) -> tuple[int, dict]:
|
||||
return cp_api_json("POST", f"{cp_url}{REDEPLOY_PATH}", token, body)
|
||||
|
||||
|
||||
def _raise_for_redeploy_result(status: int, body: dict, slugs: list[str]) -> None:
|
||||
if status != 200 or body.get("ok") is not True:
|
||||
raise RuntimeError(
|
||||
"redeploy scoped call failed for "
|
||||
f"{','.join(slugs)}: HTTP {status}, ok={body.get('ok')}"
|
||||
)
|
||||
|
||||
|
||||
def rollout_stragglers(enumerated: list[str], results: list[dict]) -> list[str]:
|
||||
"""Return every enumerated tenant NOT proven on the target build.
|
||||
|
||||
A straggler is any tenant the rollout was supposed to cover that the
|
||||
CP could not verify is running the target image tag — whether it
|
||||
errored, was skipped, or SSM-succeeded onto the wrong image
|
||||
(internal#724). CP marks each per-tenant result row with
|
||||
``verified_on_target`` (the REDEPLOY_RUNNING_IMAGE docker-inspect
|
||||
proof). A tenant enumerated for the rollout but absent from the
|
||||
result set (no batch ever ran it) is also a straggler — that is the
|
||||
exact agents-team silent-skip class.
|
||||
|
||||
Backward-compat: an OLDER CP that doesn't emit ``verified_on_target``
|
||||
yet returns rows without the key. Treat a missing key as verified so
|
||||
this surfacing degrades to the previous (ok-based) behavior against an
|
||||
un-upgraded CP, rather than failing every deploy spuriously. Once the
|
||||
CP fix is deployed the key is always present and real stragglers are
|
||||
caught.
|
||||
"""
|
||||
|
||||
verified: set[str] = set()
|
||||
for row in results:
|
||||
if str(row.get("ssm_status") or "") == "DryRun":
|
||||
continue
|
||||
slug = str(row.get("slug") or "").strip()
|
||||
if not slug:
|
||||
continue
|
||||
# Missing key (old CP) => assume verified; present key is authoritative.
|
||||
if "verified_on_target" not in row or row.get("verified_on_target"):
|
||||
verified.add(slug)
|
||||
return sorted(s for s in dict.fromkeys(enumerated) if s not in verified)
|
||||
|
||||
|
||||
def assert_full_coverage(enumerated: list[str], aggregate: dict, dry_run: bool) -> None:
|
||||
"""Fail the rollout if any enumerated tenant is not on the target build.
|
||||
|
||||
This is the no-silent-skip gate (internal#724). A dry run proves
|
||||
nothing landed, so coverage is not asserted for it.
|
||||
"""
|
||||
|
||||
if dry_run:
|
||||
return
|
||||
stragglers = rollout_stragglers(enumerated, aggregate.get("results") or [])
|
||||
if stragglers:
|
||||
msg = (
|
||||
f"incomplete rollout: {len(stragglers)} tenant(s) not verified on target "
|
||||
f"after redeploy-fleet: {', '.join(stragglers)} "
|
||||
f"(enumerated {len(set(enumerated))})"
|
||||
)
|
||||
aggregate["ok"] = False
|
||||
aggregate["error"] = msg
|
||||
aggregate["stragglers"] = stragglers
|
||||
raise RolloutFailed(msg, aggregate)
|
||||
|
||||
|
||||
def execute_scoped_rollout(
|
||||
plan: dict,
|
||||
token: str,
|
||||
list_slugs=plan_rollout_slugs,
|
||||
redeploy=redeploy_scoped,
|
||||
sleep=time.sleep,
|
||||
) -> dict:
|
||||
cp_url = plan["cp_url"]
|
||||
base_body = plan["body"]
|
||||
all_slugs = list_slugs(cp_url, token, base_body)
|
||||
batch_size = int(base_body.get("batch_size") or 1)
|
||||
canary_slug = str(base_body.get("canary_slug") or "").strip()
|
||||
dry_run = bool(base_body.get("dry_run"))
|
||||
aggregate = {"ok": True, "results": []}
|
||||
|
||||
if canary_slug:
|
||||
if canary_slug not in all_slugs:
|
||||
raise RuntimeError(f"configured canary slug {canary_slug!r} is not a running tenant")
|
||||
body = scoped_redeploy_body(base_body, [canary_slug])
|
||||
print(f"POST {cp_url}{REDEPLOY_PATH} only_slugs={','.join(body['only_slugs'])}")
|
||||
status, resp = redeploy(cp_url, token, body)
|
||||
aggregate["results"].extend(resp.get("results") or [])
|
||||
try:
|
||||
_raise_for_redeploy_result(status, resp, [canary_slug])
|
||||
except RuntimeError as exc:
|
||||
aggregate["ok"] = False
|
||||
aggregate["error"] = str(exc)
|
||||
raise RolloutFailed(str(exc), aggregate) from exc
|
||||
soak_seconds = int(base_body.get("soak_seconds") or 0)
|
||||
if soak_seconds > 0 and not dry_run:
|
||||
print(f"Canary passed; soaking locally for {soak_seconds}s")
|
||||
sleep(soak_seconds)
|
||||
|
||||
remaining = [slug for slug in all_slugs if slug != canary_slug]
|
||||
for group in chunks(remaining, batch_size):
|
||||
body = scoped_redeploy_body(base_body, group)
|
||||
print(f"POST {cp_url}{REDEPLOY_PATH} only_slugs={','.join(group)}")
|
||||
status, resp = redeploy(cp_url, token, body)
|
||||
aggregate["results"].extend(resp.get("results") or [])
|
||||
try:
|
||||
_raise_for_redeploy_result(status, resp, group)
|
||||
except RuntimeError as exc:
|
||||
aggregate["ok"] = False
|
||||
aggregate["error"] = str(exc)
|
||||
raise RolloutFailed(str(exc), aggregate) from exc
|
||||
|
||||
# No-silent-skip coverage gate (internal#724): every enumerated tenant
|
||||
# must be PROVEN on the target build. A per-tenant HTTP-200/ok response
|
||||
# is not proof — a tenant that SSM-succeeded but stayed on the old tag,
|
||||
# or one enumerated but never batched, is a straggler. Surfacing it as
|
||||
# a RolloutFailed makes the deploy step exit non-zero instead of
|
||||
# silently reporting success (the exact agents-team failure mode).
|
||||
assert_full_coverage(all_slugs, aggregate, dry_run)
|
||||
|
||||
return aggregate
|
||||
|
||||
|
||||
def rollout_from_plan_file(plan_path: str, response_path: str, env: dict[str, str]) -> None:
|
||||
token = env.get("CP_ADMIN_API_TOKEN", "").strip()
|
||||
if not token:
|
||||
raise ValueError("CP_ADMIN_API_TOKEN is required for production auto-deploy")
|
||||
with open(plan_path, "r", encoding="utf-8") as fh:
|
||||
plan = json.load(fh)
|
||||
if not plan.get("enabled"):
|
||||
raise RuntimeError("production auto-deploy plan is disabled")
|
||||
try:
|
||||
response = execute_scoped_rollout(plan, token)
|
||||
except RolloutFailed as exc:
|
||||
response = exc.response
|
||||
with open(response_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(response, fh, sort_keys=True)
|
||||
fh.write("\n")
|
||||
raise
|
||||
with open(response_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(response, fh, sort_keys=True)
|
||||
fh.write("\n")
|
||||
|
||||
|
||||
def _api_json(url: str, token: str) -> dict:
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
return json.loads(resp.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")[:500]
|
||||
raise RuntimeError(f"GET {url} -> HTTP {exc.code}: {body}") from exc
|
||||
|
||||
|
||||
def _api_json_optional(url: str, token: str) -> tuple[int, dict | None]:
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
return resp.status, json.loads(resp.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return exc.code, None
|
||||
body = exc.read().decode("utf-8", errors="replace")[:300]
|
||||
print(f"::warning::GET {url} -> HTTP {exc.code}: {body}", file=sys.stderr)
|
||||
return exc.code, None
|
||||
|
||||
|
||||
def live_disable_flag(env: dict[str, str]) -> str:
|
||||
"""Return a live disable value from Gitea variables when readable.
|
||||
|
||||
Gitea evaluates `${{ vars.* }}` once when the job starts. This API read is
|
||||
the emergency re-check immediately before production side effects.
|
||||
"""
|
||||
|
||||
token = env.get("GITEA_TOKEN", "").strip()
|
||||
if not token:
|
||||
return ""
|
||||
host = env.get("GITEA_HOST", "git.moleculesai.app")
|
||||
repo = env.get("GITHUB_REPOSITORY", "molecule-ai/molecule-core")
|
||||
variable = quote("PROD_AUTO_DEPLOY_DISABLED", safe="")
|
||||
url = f"https://{host}/api/v1/repos/{repo}/actions/variables/{variable}"
|
||||
status, body = _api_json_optional(url, token)
|
||||
if status != 200 or not isinstance(body, dict):
|
||||
return ""
|
||||
return str(body.get("data") or body.get("value") or "")
|
||||
|
||||
|
||||
def assert_not_disabled(env: dict[str, str]) -> None:
|
||||
plan = build_plan(env)
|
||||
if not plan.get("enabled"):
|
||||
raise RuntimeError(plan.get("disabled_reason", "production auto-deploy disabled"))
|
||||
live_value = live_disable_flag(env)
|
||||
if truthy_flag(live_value):
|
||||
raise RuntimeError(f"PROD_AUTO_DEPLOY_DISABLED={live_value} (live Gitea variable)")
|
||||
|
||||
|
||||
def wait_for_ci_context(env: dict[str, str]) -> str:
|
||||
host = env.get("GITEA_HOST", "git.moleculesai.app")
|
||||
repo = env.get("GITHUB_REPOSITORY", "molecule-ai/molecule-core")
|
||||
sha = env.get("GITHUB_SHA", "").strip()
|
||||
token = env.get("GITEA_TOKEN", "").strip()
|
||||
contexts = required_contexts(env)
|
||||
interval = _int_env(env, "CI_STATUS_POLL_INTERVAL_SECONDS", 15)
|
||||
timeout = _int_env(env, "CI_STATUS_TIMEOUT_SECONDS", 1800)
|
||||
|
||||
if not sha:
|
||||
raise ValueError("GITHUB_SHA is required")
|
||||
if not token:
|
||||
raise ValueError("GITEA_TOKEN is required to wait for CI status")
|
||||
|
||||
url = f"https://{host}/api/v1/repos/{repo}/commits/{sha}/status"
|
||||
deadline = time.time() + timeout
|
||||
last_states: dict[str, str] = {}
|
||||
while time.time() <= deadline:
|
||||
body = _api_json(url, token)
|
||||
statuses = body.get("statuses") or []
|
||||
states = {context: ci_context_state(statuses, context) for context in contexts}
|
||||
for context, state in states.items():
|
||||
if state != last_states.get(context):
|
||||
print(f"CI context {context!r}: {state}", file=sys.stderr)
|
||||
last_states = states
|
||||
|
||||
failures = [
|
||||
f"{context}={state}"
|
||||
for context, state in states.items()
|
||||
if context_is_terminal_failure(state)
|
||||
]
|
||||
if failures:
|
||||
raise RuntimeError(
|
||||
"Required CI context failed; refusing production deploy: "
|
||||
+ ", ".join(failures)
|
||||
)
|
||||
if all(context_is_satisfied(state) for state in states.values()):
|
||||
return "success"
|
||||
time.sleep(interval)
|
||||
last = ", ".join(f"{context}={state}" for context, state in last_states.items()) or "none"
|
||||
raise TimeoutError(f"Timed out waiting {timeout}s for required CI contexts; last_states={last}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("plan", help="print production deploy plan as JSON")
|
||||
sub.add_parser("assert-enabled", help="fail if production deploy is currently disabled")
|
||||
sub.add_parser("wait-ci", help="block until required CI context is green")
|
||||
rollout_parser = sub.add_parser("rollout", help="execute canary-first scoped production rollout")
|
||||
rollout_parser.add_argument("--plan", required=True, help="path to prod-auto-deploy plan JSON")
|
||||
rollout_parser.add_argument("--response", required=True, help="path to write aggregate response JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
if args.command == "plan":
|
||||
print(json.dumps(build_plan(dict(os.environ)), sort_keys=True))
|
||||
return 0
|
||||
if args.command == "assert-enabled":
|
||||
assert_not_disabled(dict(os.environ))
|
||||
return 0
|
||||
if args.command == "wait-ci":
|
||||
wait_for_ci_context(dict(os.environ))
|
||||
return 0
|
||||
if args.command == "rollout":
|
||||
rollout_from_plan_file(args.plan, args.response, dict(os.environ))
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001 - CLI should render operator-friendly errors.
|
||||
print(f"::error::{exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+13
-154
@@ -1,5 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC2016,SC2329
|
||||
# review-check — evaluate whether a PR satisfies a single team-review gate.
|
||||
#
|
||||
# RFC#324 Step 1 of 5 — qa-review + security-review check workflows.
|
||||
@@ -12,7 +11,6 @@
|
||||
# ≥ 1 review on the PR where:
|
||||
# • state == APPROVED
|
||||
# • review.dismissed == false
|
||||
# • review.official != false (excludes draft/mis-filed APPROVED reviews)
|
||||
# • review.user.login != PR.user.login (non-author)
|
||||
# • review.user.login ∈ team-members
|
||||
#
|
||||
@@ -62,7 +60,6 @@
|
||||
# Optional:
|
||||
# REVIEW_CHECK_DEBUG=1 — per-API-call diagnostic lines
|
||||
# REVIEW_CHECK_STRICT=1 — also require review.commit_id == pr.head.sha
|
||||
# DEFAULT_BRANCH=main — branch this gate protects; non-default-base PRs no-op
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -94,7 +91,7 @@ API="https://${GITEA_HOST}/api/v1"
|
||||
# secret token value in the process table for any process to read via
|
||||
# /proc/<pid>/cmdline or ps -ef). The curl config file is read by curl
|
||||
# itself and never appears in the argv of the curl subprocess.
|
||||
CURL_AUTH_FILE=$(mktemp "${TMPDIR:-/tmp}/curl-auth.XXXXXX")
|
||||
CURL_AUTH_FILE=$(mktemp -p /tmp curl-auth.XXXXXX)
|
||||
chmod 600 "$CURL_AUTH_FILE"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$CURL_AUTH_FILE"
|
||||
|
||||
@@ -102,12 +99,10 @@ printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$CURL_AUTH_FILE"
|
||||
# (bash trap 'function' EXIT expands variables at trap-fire time, not def time).
|
||||
PR_JSON=$(mktemp)
|
||||
REVIEWS_JSON=$(mktemp)
|
||||
COMMENTS_JSON=$(mktemp)
|
||||
TEAM_PROBE_TMP=$(mktemp)
|
||||
NA_STATUSES_TMP="" # declared here so cleanup() always has the var
|
||||
|
||||
cleanup() {
|
||||
rm -f "$CURL_AUTH_FILE" "$PR_JSON" "$REVIEWS_JSON" "$COMMENTS_JSON" "$TEAM_PROBE_TMP" "${NA_STATUSES_TMP-}"
|
||||
rm -f "$CURL_AUTH_FILE" "$PR_JSON" "$REVIEWS_JSON" "$TEAM_PROBE_TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
@@ -129,65 +124,18 @@ if [ "$HTTP_CODE" != "200" ]; then
|
||||
fi
|
||||
PR_AUTHOR=$(jq -r '.user.login // ""' "$PR_JSON")
|
||||
PR_HEAD_SHA=$(jq -r '.head.sha // ""' "$PR_JSON")
|
||||
PR_BASE_REF=$(jq -r '.base.ref // ""' "$PR_JSON")
|
||||
PR_BASE_SHA=$(jq -r '.base.sha // ""' "$PR_JSON")
|
||||
PR_STATE=$(jq -r '.state // ""' "$PR_JSON")
|
||||
DEFAULT_BRANCH="${DEFAULT_BRANCH:-main}"
|
||||
debug "pr_author=${PR_AUTHOR} pr_head=${PR_HEAD_SHA:0:7} pr_base=${PR_BASE_REF} pr_state=${PR_STATE}"
|
||||
debug "pr_author=${PR_AUTHOR} pr_head=${PR_HEAD_SHA:0:7} pr_state=${PR_STATE}"
|
||||
|
||||
if [ "$PR_STATE" != "open" ]; then
|
||||
echo "::notice::PR ${PR_NUMBER} is ${PR_STATE} — exiting 0 (closed PRs do not gate)"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$PR_HEAD_SHA" = "$PR_BASE_SHA" ]; then
|
||||
echo "::notice::PR ${PR_NUMBER} has no diff (head == base) — exiting 0 (empty PRs do not gate)"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$PR_BASE_REF" != "$DEFAULT_BRANCH" ]; then
|
||||
echo "::notice::PR ${PR_NUMBER} targets ${PR_BASE_REF:-<unknown>} not ${DEFAULT_BRANCH} — ${TEAM}-review gate not applicable"
|
||||
exit 0
|
||||
fi
|
||||
if [ -z "$PR_AUTHOR" ] || [ -z "$PR_HEAD_SHA" ]; then
|
||||
echo "::error::PR ${PR_NUMBER} missing user.login or head.sha — webhook payload malformed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- RFC#324 §N/A follow-up: check N/A declarations status ---
|
||||
# sop-checklist.py posts `sop-checklist / na-declarations (pull_request)`
|
||||
# status when a peer posts /sop-n/a <gate>. If our gate is declared N/A,
|
||||
# the requirement for a Gitea APPROVE review is waived.
|
||||
NA_STATUSES_TMP=$(mktemp)
|
||||
HTTP_CODE=$(curl -sS -o "$NA_STATUSES_TMP" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/statuses/${PR_HEAD_SHA}")
|
||||
debug "statuses/${PR_HEAD_SHA} → HTTP ${HTTP_CODE}"
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
# Gitea returns statuses as array; look for the na-declarations context.
|
||||
# jq: find all statuses where context == "sop-checklist / na-declarations (pull_request)"
|
||||
# and state == "success". Extract the description field.
|
||||
NA_DESC=$(jq -r '
|
||||
.[] |
|
||||
select(.context == "sop-checklist / na-declarations (pull_request)") |
|
||||
select(.state == "success") |
|
||||
.description
|
||||
' "$NA_STATUSES_TMP" 2>/dev/null | head -1)
|
||||
|
||||
if [ -n "$NA_DESC" ] && [ "$NA_DESC" != "null" ]; then
|
||||
debug "na-declarations status found: ${NA_DESC}"
|
||||
# Check if our gate appears in the N/A description.
|
||||
# The description format is "N/A: qa-review, security-review" or similar.
|
||||
if echo "$NA_DESC" | grep -iq "\\b${TEAM}-review\\b"; then
|
||||
echo "::notice::${TEAM}-review N/A — gate declared not-applicable via /sop-n/a: ${NA_DESC}"
|
||||
echo "::notice::PR ${PR_NUMBER} passes ${TEAM}-review via N/A declaration"
|
||||
rm -f "$NA_STATUSES_TMP"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
else
|
||||
debug "could not fetch statuses (HTTP ${HTTP_CODE}) — proceeding with normal eval"
|
||||
fi
|
||||
rm -f "$NA_STATUSES_TMP"
|
||||
|
||||
# --- Fetch all reviews on the PR ---
|
||||
HTTP_CODE=$(curl -sS -o "$REVIEWS_JSON" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews")
|
||||
@@ -202,7 +150,6 @@ fi
|
||||
JQ_FILTER='.[]
|
||||
| select(.state == "APPROVED")
|
||||
| select(.dismissed != true)
|
||||
| select(.official != false)
|
||||
| select(.user.login != $author)'
|
||||
if [ "${REVIEW_CHECK_STRICT:-}" = "1" ]; then
|
||||
JQ_FILTER="${JQ_FILTER}
|
||||
@@ -211,82 +158,11 @@ fi
|
||||
JQ_FILTER="${JQ_FILTER}
|
||||
| .user.login"
|
||||
|
||||
REVIEW_CANDIDATES=$(jq -r --arg author "$PR_AUTHOR" --arg head "$PR_HEAD_SHA" "$JQ_FILTER" "$REVIEWS_JSON" | sort -u)
|
||||
debug "candidate non-author approvers: $(echo "$REVIEW_CANDIDATES" | tr '\n' ' ')"
|
||||
CANDIDATES=$(jq -r --arg author "$PR_AUTHOR" --arg head "$PR_HEAD_SHA" "$JQ_FILTER" "$REVIEWS_JSON" | sort -u)
|
||||
debug "candidate non-author approvers: $(echo "$CANDIDATES" | tr '\n' ' ')"
|
||||
|
||||
if [ -z "$REVIEW_CANDIDATES" ]; then
|
||||
# --- Guardrail (internal#503): explain the most common false
|
||||
# "no candidates" red. Gitea's review event enum is EXACTLY
|
||||
# APPROVED/REQUEST_CHANGES/COMMENT/PENDING. A wrong value ("APPROVE",
|
||||
# lowercase, ...) is silently accepted (HTTP 200) and stored as
|
||||
# state=PENDING. A correctly-started draft review has an EMPTY body;
|
||||
# a NON-empty body + state==PENDING by a non-author == an intended
|
||||
# verdict mis-filed by a wrong event string. Surface it actionably.
|
||||
# This does NOT change the gate result (still fail-closed below) — it
|
||||
# only converts a mystery red into a named, self-fixing error.
|
||||
MISFILED_FILTER='.[]
|
||||
| select(.state == "PENDING")
|
||||
| select(.dismissed != true)
|
||||
| select(.user.login != $author)
|
||||
| select(((.body // "") | gsub("^\\s+|\\s+$";"") | length) > 0)
|
||||
| "\(.id)\t\(.user.login)"'
|
||||
MISFILED=$(jq -r --arg author "$PR_AUTHOR" "$MISFILED_FILTER" "$REVIEWS_JSON" 2>/dev/null || true)
|
||||
if [ -n "$MISFILED" ]; then
|
||||
echo "::error::${TEAM}-review: non-author review(s) were SUBMITTED but stored as PENDING — almost certainly the wrong Gitea review event string (internal#503)."
|
||||
echo "::error::Gitea accepts ONLY the exact enum APPROVED / REQUEST_CHANGES / COMMENT. 'APPROVE' or lowercase is silently (HTTP 200) filed as PENDING and is invisible to this gate."
|
||||
printf '%s\n' "$MISFILED" | while IFS="$(printf '\t')" read -r _rid _rl; do
|
||||
[ -n "${_rid:-}" ] && echo "::error:: review id=${_rid} by '${_rl}': RE-SUBMIT via POST ${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews with {\"event\":\"APPROVED\"} (correct enum) — do NOT edit the DB."
|
||||
done
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
# --- Fallback/extension (internal#348): check issue comments for agent-approval ---
|
||||
# core-qa-agent and core-security-agent can approve via issue comments. Always
|
||||
# include comment candidates, even if the reviews API returned approvals for a
|
||||
# different team; team membership below is the authoritative filter.
|
||||
COMMENT_CANDIDATES=""
|
||||
AGENT_PATTERN=""
|
||||
case "$TEAM" in
|
||||
qa) AGENT_PATTERN="\\[core-qa-agent\\]" ;;
|
||||
security) AGENT_PATTERN="\\[core-security-agent\\]" ;;
|
||||
esac
|
||||
HTTP_CODE=$(curl -sS -o "$COMMENTS_JSON" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/issues/${PR_NUMBER}/comments")
|
||||
debug "GET /issues/${PR_NUMBER}/comments → HTTP ${HTTP_CODE}"
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
# JQ expression: select non-author comments that match either the
|
||||
# agent-prefix pattern (case-insensitive) OR a generic approval keyword.
|
||||
JQ_APPROVALS='
|
||||
.[] |
|
||||
select(.user.login != $author) |
|
||||
. as $cmt |
|
||||
if ($agent_pattern | length) > 0 and ($cmt.body // "" | test($agent_pattern; "i")) then
|
||||
$cmt.user.login
|
||||
elif ($cmt.body // "" | test("\\b(APPROVED|LGTM|ACCEPTED)\\b"; "i")) then
|
||||
$cmt.user.login
|
||||
else
|
||||
empty
|
||||
end
|
||||
'
|
||||
COMMENT_CANDIDATES=$(jq -r \
|
||||
--arg author "$PR_AUTHOR" \
|
||||
--arg agent_pattern "$AGENT_PATTERN" \
|
||||
"$JQ_APPROVALS" \
|
||||
"$COMMENTS_JSON" 2>/dev/null | sort -u)
|
||||
debug "comment-based approval candidates: $(echo "$COMMENT_CANDIDATES" | tr '\n' ' ')"
|
||||
|
||||
if [ -n "$COMMENT_CANDIDATES" ]; then
|
||||
echo "::notice::${TEAM}-review: found $(echo "$COMMENT_CANDIDATES" | wc -w | xargs) comment-based approval candidate(s) — verifying team membership..."
|
||||
fi
|
||||
else
|
||||
debug "could not fetch issue comments (HTTP ${HTTP_CODE})"
|
||||
fi
|
||||
|
||||
CANDIDATES=$(printf '%s\n%s\n' "$REVIEW_CANDIDATES" "$COMMENT_CANDIDATES" | sed '/^$/d' | sort -u)
|
||||
|
||||
if [ -z "${CANDIDATES:-}" ]; then
|
||||
echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (no candidates from reviews API or issue comments)"
|
||||
if [ -z "$CANDIDATES" ]; then
|
||||
echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (no candidates yet)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -296,15 +172,7 @@ fi
|
||||
# 403 → token owner is not in this team (Gitea 1.22.6 'Must be a team
|
||||
# member' constraint — see follow-up issue for token-provisioning)
|
||||
# 404 → not a member
|
||||
# Track whether every candidate returned 403 (token owner not in team).
|
||||
# When this happens the root cause is a token-provisioning issue, not a
|
||||
# reviewer-eligibility issue — surface it clearly so ops don't waste time
|
||||
# verifying team roster (Bug C / RFC#324 follow-up).
|
||||
_ALL_CANDIDATES_403="yes"
|
||||
_CANDIDATE_COUNT=0
|
||||
|
||||
for U in $CANDIDATES; do
|
||||
_CANDIDATE_COUNT=$((_CANDIDATE_COUNT + 1))
|
||||
CODE=$(curl -sS -o "$TEAM_PROBE_TMP" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/teams/${TEAM_ID}/members/${U}")
|
||||
debug "probe ${U} in team ${TEAM} (id=${TEAM_ID}) → HTTP ${CODE}"
|
||||
@@ -314,31 +182,22 @@ for U in $CANDIDATES; do
|
||||
exit 0
|
||||
;;
|
||||
403)
|
||||
# Token owner is not in the team being probed; Gitea 1.22.6 refuses
|
||||
# to confirm membership in this case. Do NOT hard-fail the gate on a
|
||||
# 403 — doing so would fail the entire gate if ANY candidate triggers
|
||||
# a 403, even when other valid team-members exist. Instead skip this
|
||||
# candidate and continue checking others. If all candidates produce
|
||||
# 403 (token owner can't query any of them) the final exit fires.
|
||||
echo "::warning::team-probe for ${U} in ${TEAM} returned 403 (token owner not in ${TEAM} team — skipping; cannot confirm membership)"
|
||||
# Token owner is not in the team being probed; the API refuses to
|
||||
# confirm membership. This is the RFC#324 follow-up token-scope gap.
|
||||
# Fail closed — never grant approval on a 403; surface clearly.
|
||||
echo "::error::team-probe for ${U} in ${TEAM} returned 403 (token owner not in ${TEAM} team — RFC#324 token-scope follow-up). Cannot confirm membership; failing closed."
|
||||
cat "$TEAM_PROBE_TMP" >&2
|
||||
continue
|
||||
exit 1
|
||||
;;
|
||||
404)
|
||||
_ALL_CANDIDATES_403="no"
|
||||
debug "${U} not a member of ${TEAM}"
|
||||
;;
|
||||
*)
|
||||
_ALL_CANDIDATES_403="no"
|
||||
echo "::warning::team-probe for ${U} in ${TEAM} returned unexpected HTTP ${CODE}"
|
||||
cat "$TEAM_PROBE_TMP" >&2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$_ALL_CANDIDATES_403" = "yes" ] && [ "$_CANDIDATE_COUNT" -gt 0 ]; then
|
||||
echo "::error::${TEAM}-review FAILED — every candidate returned 403 (token owner is not a member of the ${TEAM} team). This is a TOKEN PROVISIONING issue, not a reviewer-eligibility issue. Add the token owner to the '${TEAM}' Gitea team (id=${TEAM_ID}) or use a token whose owner is already in that team."
|
||||
else
|
||||
echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (candidates: $(echo "$CANDIDATES" | tr '\n' ',' | sed 's/,$//') — none are in team)"
|
||||
fi
|
||||
echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (candidates: $(echo "$CANDIDATES" | tr '\n' ',' | sed 's/,$//') — none are in team)"
|
||||
exit 1
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-run review-check.sh for a slash-command refire and post the protected
|
||||
# pull_request status context to the PR head SHA.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${GITEA_TOKEN:?GITEA_TOKEN required}"
|
||||
: "${GITEA_HOST:?GITEA_HOST required}"
|
||||
: "${REPO:?REPO required}"
|
||||
: "${PR_NUMBER:?PR_NUMBER required}"
|
||||
: "${TEAM:?TEAM required}"
|
||||
|
||||
OWNER="${REPO%%/*}"
|
||||
NAME="${REPO##*/}"
|
||||
API="https://${GITEA_HOST}/api/v1"
|
||||
# Branch-protection requires the (pull_request_target) context variant.
|
||||
# The refire path must post the EXACT BP-required name so the gate flips.
|
||||
CONTEXT="${TEAM}-review / approved (pull_request_target)"
|
||||
TARGET_URL="https://${GITEA_HOST}/${OWNER}/${NAME}/pulls/${PR_NUMBER}"
|
||||
|
||||
authfile=$(mktemp)
|
||||
post_authfile=$(mktemp)
|
||||
prfile=$(mktemp)
|
||||
postfile=$(mktemp)
|
||||
# shellcheck disable=SC2329 # invoked by EXIT trap
|
||||
cleanup() {
|
||||
rm -f "$authfile" "$post_authfile" "$prfile" "$postfile"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
chmod 600 "$authfile" "$post_authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
# STATUS_POST_TOKEN is narrow-scoped write:repository for explicit status POST.
|
||||
# Falls back to GITEA_TOKEN for backward compatibility (e.g. local test).
|
||||
printf 'header = "Authorization: token %s"\n' "${STATUS_POST_TOKEN:-$GITEA_TOKEN}" > "$post_authfile"
|
||||
|
||||
code=$(curl -sS -o "$prfile" -w '%{http_code}' -K "$authfile" \
|
||||
"${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::GET /pulls/${PR_NUMBER} returned HTTP ${code}"
|
||||
head -c 200 "$prfile" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
head_sha=$(jq -r '.head.sha // ""' "$prfile")
|
||||
state=$(jq -r '.state // ""' "$prfile")
|
||||
if [ -z "$head_sha" ] || [ "$head_sha" = "null" ]; then
|
||||
echo "::error::Could not resolve PR head SHA for PR ${PR_NUMBER}"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$state" != "open" ]; then
|
||||
echo "::notice::PR ${PR_NUMBER} is ${state}; ${TEAM}-review refire is a no-op"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
set +e
|
||||
bash .gitea/scripts/review-check.sh
|
||||
rc=$?
|
||||
set -e
|
||||
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
status_state="success"
|
||||
description="Refired via /${TEAM}-recheck by ${COMMENT_AUTHOR:-unknown}"
|
||||
else
|
||||
status_state="failure"
|
||||
description="Refired via /${TEAM}-recheck; ${TEAM}-review failed"
|
||||
fi
|
||||
|
||||
body=$(jq -nc \
|
||||
--arg state "$status_state" \
|
||||
--arg context "$CONTEXT" \
|
||||
--arg description "$description" \
|
||||
--arg target_url "$TARGET_URL" \
|
||||
'{state:$state, context:$context, description:$description, target_url:$target_url}')
|
||||
|
||||
code=$(curl -sS -o "$postfile" -w '%{http_code}' -X POST \
|
||||
-K "$post_authfile" -H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"${API}/repos/${OWNER}/${NAME}/statuses/${head_sha}")
|
||||
if [ "$code" != "200" ] && [ "$code" != "201" ]; then
|
||||
echo "::error::POST /statuses/${head_sha} returned HTTP ${code}"
|
||||
head -c 200 "$postfile" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::posted ${status_state} for context=\"${CONTEXT}\" on sha=${head_sha}"
|
||||
exit "$rc"
|
||||
Regular → Executable
+55
-491
@@ -1,13 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
# sop-checklist — evaluate whether a PR has peer-acked each
|
||||
# sop-checklist-gate — evaluate whether a PR has peer-acked each
|
||||
# SOP-checklist item. Posts a commit-status that branch protection
|
||||
# can require.
|
||||
#
|
||||
# RFC#351 Step 2 of 6 (implementation MVP).
|
||||
#
|
||||
# Invoked by .gitea/workflows/sop-checklist.yml on:
|
||||
# - pull_request_target: [opened, edited, synchronize, reopened, labeled, unlabeled]
|
||||
# - issue_comment: [created] # edited/deleted omitted (Gitea 1.22.6 job-parsing quirk)
|
||||
# Invoked by .gitea/workflows/sop-checklist-gate.yml on:
|
||||
# - pull_request_target: [opened, edited, synchronize, reopened]
|
||||
# - issue_comment: [created, edited, deleted]
|
||||
#
|
||||
# Flow:
|
||||
# 1. Load .gitea/sop-checklist-config.yaml (from BASE ref — trusted).
|
||||
@@ -64,41 +64,11 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any, Callable, Iterator
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Address-space guardrail (RFC#369 / task #369 follow-up to mc#1242-class OOM).
|
||||
#
|
||||
# `get_issue_comments` paginates the full comment history of a PR. On
|
||||
# bot-relay-heavy PRs (e.g. mc#291, mc#1242) this can balloon past the
|
||||
# runner's cgroup memory limit and 137 the job. Cap virtual-address-space
|
||||
# at 2 GiB so the script OOMs as a `MemoryError` (catchable / surfaceable)
|
||||
# rather than a SIGKILL we can't post a status for.
|
||||
#
|
||||
# 2 GiB is generous — a 5000-comment PR with 1 KiB minimal-dicts (see
|
||||
# get_issue_comments below) fits in ~10 MiB, leaving plenty of headroom
|
||||
# for the Python runtime + urllib + json buffers.
|
||||
#
|
||||
# Skipped under pytest / dry-run where RLIMIT_AS would interfere with
|
||||
# test runner memory needs (set SOP_CHECKLIST_NO_RLIMIT=1 to opt out).
|
||||
if not os.environ.get("SOP_CHECKLIST_NO_RLIMIT"):
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_AS, (2 * 1024**3, 2 * 1024**3))
|
||||
except (ValueError, OSError):
|
||||
# macOS sometimes refuses RLIMIT_AS; not fatal — the Linux runner
|
||||
# is the only place this matters for the OOM-prevention goal.
|
||||
pass
|
||||
|
||||
# Per-comment body cap (task #369). The directive parser walks the body
|
||||
# line-by-line looking for ^/sop-ack ^/sop-revoke ^/sop-n/a markers — only
|
||||
# the first few KiB matter for that. Cap each comment body so a single
|
||||
# pasted-log comment can't push us past the cgroup limit.
|
||||
_MAX_BODY_BYTES = int(os.environ.get("SOP_CHECKLIST_MAX_BODY_BYTES") or 8 * 1024)
|
||||
from typing import Any
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -140,7 +110,7 @@ def normalize_slug(raw: str, numeric_aliases: dict[int, str] | None = None) -> s
|
||||
# for /sop-revoke (RFC#351 open question 4 — reason is captured but not
|
||||
# yet validated; future iteration may require a min-length).
|
||||
_DIRECTIVE_RE = re.compile(
|
||||
r"^[ \t]*/(sop-ack|sop-revoke|sop-n/a)[ \t]+([A-Za-z0-9_\- ]+?)(?:[ \t]+(.*))?[ \t]*$",
|
||||
r"^[ \t]*/(sop-ack|sop-revoke)[ \t]+([A-Za-z0-9_\- ]+?)(?:[ \t]+(.*))?[ \t]*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
@@ -148,21 +118,17 @@ _DIRECTIVE_RE = re.compile(
|
||||
def parse_directives(
|
||||
comment_body: str,
|
||||
numeric_aliases: dict[int, str],
|
||||
) -> tuple[list[tuple[str, str, str]], list[tuple[str, str, str]]]:
|
||||
"""Extract /sop-ack, /sop-revoke, and /sop-n/a directives from a comment body.
|
||||
) -> list[tuple[str, str, str]]:
|
||||
"""Extract /sop-ack and /sop-revoke directives from a comment body.
|
||||
|
||||
Returns (directives, na_directives) where each is a list of
|
||||
(kind, canonical_slug, note) tuples:
|
||||
kind is "sop-ack", "sop-revoke", or "sop-n/a"
|
||||
Returns a list of (kind, canonical_slug, note) tuples where:
|
||||
kind is "sop-ack" or "sop-revoke"
|
||||
canonical_slug is the normalized form (or "" if unparseable)
|
||||
note is the trailing free-text (may be "")
|
||||
The two lists are kept separate so call sites can unpack them
|
||||
directly (e.g. directives, na_directives = parse_directives(...)).
|
||||
"""
|
||||
directives: list[tuple[str, str, str]] = []
|
||||
na_directives: list[tuple[str, str, str]] = []
|
||||
out: list[tuple[str, str, str]] = []
|
||||
if not comment_body:
|
||||
return directives, na_directives
|
||||
return out
|
||||
for m in _DIRECTIVE_RE.finditer(comment_body):
|
||||
kind = m.group(1)
|
||||
raw_slug = (m.group(2) or "").strip()
|
||||
@@ -192,12 +158,8 @@ def parse_directives(
|
||||
note_from_group = (m.group(3) or "").strip()
|
||||
# If we collapsed multi-word slug into kebab and there's a
|
||||
# trailing-text group too, append it.
|
||||
entry = (kind, canonical, note_from_group)
|
||||
if kind == "sop-n/a":
|
||||
na_directives.append(entry)
|
||||
else:
|
||||
directives.append(entry)
|
||||
return directives, na_directives
|
||||
out.append((kind, canonical, note_from_group))
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -210,8 +172,8 @@ def section_marker_present(body: str, marker: str) -> bool:
|
||||
on a non-empty line (i.e. the author actually filled it in).
|
||||
|
||||
We require the marker substring AND non-whitespace content on the
|
||||
same line OR within the next non-blank line — this prevents
|
||||
trivially-empty checklists like:
|
||||
same line OR within the next line — this prevents trivially-empty
|
||||
checklists like:
|
||||
|
||||
## SOP-Checklist
|
||||
- [ ] **Comprehensive testing performed**:
|
||||
@@ -220,18 +182,9 @@ def section_marker_present(body: str, marker: str) -> bool:
|
||||
from auto-passing the section-present check. The peer-ack is still
|
||||
required, but answering with empty content is captured as a soft
|
||||
finding via the section-present test alone.
|
||||
|
||||
NOTE: we scan forward through blank lines (the markdown-header pattern
|
||||
is ## Header\\n\\ncontent) so that a header + blank-line + content
|
||||
structure still satisfies the check. The backward checkbox fallback
|
||||
catches inline markers without a preceding checkbox (mc#1099).
|
||||
"""
|
||||
if not body or not marker:
|
||||
return False
|
||||
# Strip trailing whitespace so the blank-line scan below can find
|
||||
# content that appears on the very last line of the body (without
|
||||
# being misled by a trailing \n or spaces).
|
||||
body = body.rstrip()
|
||||
body_lower = body.lower()
|
||||
marker_lower = marker.lower()
|
||||
idx = body_lower.find(marker_lower)
|
||||
@@ -247,44 +200,13 @@ def section_marker_present(body: str, marker: str) -> bool:
|
||||
stripped = re.sub(r"[\s\*:\-\[\]]+", "", line)
|
||||
if stripped:
|
||||
return True
|
||||
# Fall through: scan forward, skipping blank-only lines, until we find
|
||||
# non-empty content or run out of body. Handles:
|
||||
# ## Header ← marker line (empty after marker)
|
||||
# ← blank line (skipped)
|
||||
# - actual content ← found
|
||||
pos = line_end
|
||||
while True:
|
||||
# Skip the current newline and any additional newlines (blank lines).
|
||||
while pos < len(body) and body[pos] == "\n":
|
||||
pos += 1
|
||||
if pos >= len(body):
|
||||
break
|
||||
line_end = body.find("\n", pos)
|
||||
if line_end < 0:
|
||||
line_end = len(body)
|
||||
line = body[pos:line_end]
|
||||
stripped = re.sub(r"[\s\*:\-\[\]]+", "", line)
|
||||
if stripped:
|
||||
return True
|
||||
pos = line_end
|
||||
# Last resort: the marker may appear mid-sentence (e.g.
|
||||
# **Memory/saved-feedback consulted**: No applicable...).
|
||||
# Search backward within the CURRENT LINE only (not preceding lines)
|
||||
# to find a checkbox on the same line before the marker text.
|
||||
# mc#1099 follow-up: memory-consulted detection was failing because
|
||||
# the checkbox was on the same line before the inline marker.
|
||||
_CHECKBOX_RE = re.compile(r"- \[[ x\]]|<input", re.IGNORECASE)
|
||||
line_start = body.rfind("\n", 0, idx) + 1 # 0 if no newline before idx
|
||||
before = body[line_start:idx]
|
||||
m = _CHECKBOX_RE.search(before)
|
||||
if not m:
|
||||
return False
|
||||
# Require meaningful content between the checkbox and the marker text
|
||||
# (markdown formatting like ** or * must also be stripped).
|
||||
# If only whitespace/markdown chars remain, the checkbox line is empty.
|
||||
between = before[m.end() :]
|
||||
stripped_between = re.sub(r"[\s\*:#\[\]_\-]+", "", between)
|
||||
return bool(stripped_between)
|
||||
# Fall through: check the NEXT line (multi-line answers).
|
||||
next_line_end = body.find("\n", line_end + 1)
|
||||
if next_line_end < 0:
|
||||
next_line_end = len(body)
|
||||
next_line = body[line_end + 1:next_line_end]
|
||||
stripped_next = re.sub(r"[\s\*:\-\[\]]+", "", next_line)
|
||||
return bool(stripped_next)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -298,7 +220,6 @@ def compute_ack_state(
|
||||
items_by_slug: dict[str, dict[str, Any]],
|
||||
numeric_aliases: dict[int, str],
|
||||
team_membership_probe: "callable[[str, list[str]], list[str]]",
|
||||
high_risk: bool = False,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Compute per-item ack state.
|
||||
|
||||
@@ -328,7 +249,7 @@ def compute_ack_state(
|
||||
user = (c.get("user") or {}).get("login", "")
|
||||
if not user:
|
||||
continue
|
||||
for kind, slug, _note in parse_directives(body, numeric_aliases)[0]:
|
||||
for kind, slug, _note in parse_directives(body, numeric_aliases):
|
||||
if not slug:
|
||||
unparseable_per_user[user] = unparseable_per_user.get(user, 0) + 1
|
||||
continue
|
||||
@@ -338,6 +259,7 @@ def compute_ack_state(
|
||||
# Filter out self-acks and unknown slugs.
|
||||
ackers_per_slug: dict[str, list[str]] = {s: [] for s in items_by_slug}
|
||||
rejected_self: dict[str, list[str]] = {s: [] for s in items_by_slug}
|
||||
rejected_unknown: dict[str, list[str]] = {s: [] for s in items_by_slug}
|
||||
pending_team_check: dict[str, list[str]] = {s: [] for s in items_by_slug}
|
||||
|
||||
for (user, slug), kind in latest_directive.items():
|
||||
@@ -360,16 +282,11 @@ def compute_ack_state(
|
||||
for slug, candidates in pending_team_check.items():
|
||||
if not candidates:
|
||||
continue
|
||||
# Risk-class-aware required-teams resolution (RFC#450 Option C):
|
||||
# high-risk PRs use `required_teams_high_risk` (when set on the
|
||||
# item); default class uses `required_teams`. The probe closure
|
||||
# is built with the same high_risk flag so the two reads are
|
||||
# always consistent (both sites share `resolve_required_teams`).
|
||||
required = resolve_required_teams(items_by_slug[slug], high_risk)
|
||||
required = items_by_slug[slug]["required_teams"]
|
||||
approved = team_membership_probe(slug, candidates) # returns subset
|
||||
rejected_not_in_team[slug] = [u for u in candidates if u not in approved]
|
||||
ackers_per_slug[slug] = approved
|
||||
# Stash resolved teams for description rendering.
|
||||
# Stash required teams for description rendering.
|
||||
items_by_slug[slug]["_required_resolved"] = required
|
||||
|
||||
return {
|
||||
@@ -384,63 +301,6 @@ def compute_ack_state(
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# N/A-gate evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_na_state(
|
||||
comments: list[dict[str, Any]],
|
||||
author: str,
|
||||
na_gates: dict[str, Any],
|
||||
probe: Callable[[str, list[str]], list[str]],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Evaluate which N/A gates have a valid declaration from a team member.
|
||||
|
||||
Returns dict[gate_name, dict] where each dict has:
|
||||
declared: bool — at least one valid non-author team-member declared N/A
|
||||
decl_ackers: list[str] — usernames who declared this gate N/A
|
||||
rejected: dict with keys:
|
||||
not_in_team: list[str] — users who tried but aren't in required teams
|
||||
"""
|
||||
# Build per-user latest N/A directive (most-recent wins per RFC#324).
|
||||
latest_na: dict[str, tuple[str, str]] = {} # user → (gate, note)
|
||||
for c in comments:
|
||||
body = c.get("body", "") or ""
|
||||
user = (c.get("user") or {}).get("login", "")
|
||||
if not user:
|
||||
continue
|
||||
for kind, gate, note in parse_directives(body, {})[1]:
|
||||
# [1] = na_directives only
|
||||
if gate in na_gates:
|
||||
latest_na[user] = (gate, note)
|
||||
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for gate, gate_cfg in na_gates.items():
|
||||
result[gate] = {
|
||||
"declared": False,
|
||||
"decl_ackers": [],
|
||||
"rejected": {"not_in_team": []},
|
||||
}
|
||||
decl_ackers: list[str] = []
|
||||
not_in_team: list[str] = []
|
||||
for user, (g, _note) in latest_na.items():
|
||||
if g != gate:
|
||||
continue
|
||||
if user == author:
|
||||
continue # authors cannot self-declare N/A
|
||||
approved = probe(gate, [user])
|
||||
if approved:
|
||||
decl_ackers.append(user)
|
||||
else:
|
||||
not_in_team.append(user)
|
||||
result[gate]["declared"] = bool(decl_ackers)
|
||||
result[gate]["decl_ackers"] = decl_ackers
|
||||
result[gate]["rejected"]["not_in_team"] = not_in_team
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gitea API client
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -489,35 +349,16 @@ class GiteaClient:
|
||||
raise RuntimeError(f"GET pulls/{pr} → HTTP {code}: {data!r}")
|
||||
return data
|
||||
|
||||
def iter_issue_comments(
|
||||
self, owner: str, repo: str, issue: int, page_size: int = 50
|
||||
) -> Iterator[dict[str, Any]]:
|
||||
"""Stream comments page-by-page, yielding ONE minimal-dict per comment.
|
||||
|
||||
Each yielded comment carries ONLY the fields the gate actually reads
|
||||
— `{"user": {"login": str}, "body": str}` — and DROPS the much
|
||||
larger Gitea-API extras (html_url, pull_request_url, issue_url,
|
||||
assets, created_at, updated_at, id, original_author_*).
|
||||
|
||||
Memory motivation (task #369 / mc#1242-class OOM): full Gitea
|
||||
comment dicts are ~2 KiB median + ~3 KiB p95. On PRs with several
|
||||
thousand bot-relay comments the eager `list[full_dict]` shape used
|
||||
previously pushed runner anon-rss past the cgroup limit. The
|
||||
minimal-dict shape is ~10-20x smaller (typically ~50-100B Python
|
||||
overhead + the body string).
|
||||
|
||||
The two downstream consumers (`compute_ack_state`,
|
||||
`compute_na_state`) each iterate the comment list exactly once and
|
||||
read only `body` + `user.login`, so dropping every other field is
|
||||
safe. They still receive `list[dict[str, Any]]`-shaped objects so
|
||||
the test fixtures (which already used the minimal shape) keep
|
||||
working with no fixture changes.
|
||||
"""
|
||||
def get_issue_comments(
|
||||
self, owner: str, repo: str, issue: int
|
||||
) -> list[dict[str, Any]]:
|
||||
# Paginate. Gitea default page size 50.
|
||||
out: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
while True:
|
||||
code, data = self._req(
|
||||
"GET",
|
||||
f"/repos/{owner}/{repo}/issues/{issue}/comments?limit={page_size}&page={page}",
|
||||
f"/repos/{owner}/{repo}/issues/{issue}/comments?limit=50&page={page}",
|
||||
)
|
||||
if code != 200:
|
||||
raise RuntimeError(
|
||||
@@ -525,41 +366,10 @@ class GiteaClient:
|
||||
)
|
||||
if not data:
|
||||
break
|
||||
for c in data:
|
||||
# Minimal projection — drop ALL fields the gate doesn't read.
|
||||
user_login = ((c.get("user") or {}).get("login") or "") if isinstance(c, dict) else ""
|
||||
body = (c.get("body") if isinstance(c, dict) else "") or ""
|
||||
# Body-size guardrail: huge comments (e.g. pasted CI logs) can
|
||||
# individually be MiBs. The directive parser only needs the
|
||||
# first ~8 KiB to find /sop-ack /sop-revoke /sop-n/a markers
|
||||
# — anything past that is filler. Truncate at 8 KiB so a
|
||||
# single oversized comment can't OOM the runner.
|
||||
if len(body) > _MAX_BODY_BYTES:
|
||||
body = body[:_MAX_BODY_BYTES]
|
||||
yield {"user": {"login": user_login}, "body": body}
|
||||
if len(data) < page_size:
|
||||
out.extend(data)
|
||||
if len(data) < 50:
|
||||
break
|
||||
page += 1
|
||||
|
||||
def get_issue_comments(
|
||||
self,
|
||||
owner: str,
|
||||
repo: str,
|
||||
issue: int,
|
||||
max_comments: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Paginate + collect minimal comment dicts. See `iter_issue_comments`
|
||||
for the per-comment shape and the OOM-prevention rationale.
|
||||
|
||||
`max_comments` (optional, default unbounded): hard cap. When the cap
|
||||
is hit we stop fetching further pages and the caller surfaces a
|
||||
soft 'skipping due to volume' status (see main()).
|
||||
"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for c in self.iter_issue_comments(owner, repo, issue):
|
||||
out.append(c)
|
||||
if max_comments is not None and len(out) >= max_comments:
|
||||
break
|
||||
return out
|
||||
|
||||
def resolve_team_id(self, org: str, team_name: str) -> int | None:
|
||||
@@ -636,11 +446,8 @@ def load_config(path: str) -> dict[str, Any]:
|
||||
dep by keeping the config shape constrained.
|
||||
"""
|
||||
try:
|
||||
# yaml is an optional dep; the canonical loader is used when available,
|
||||
# but the SOP runs on runners that may not have PyYAML installed. The
|
||||
# fallback _load_config_minimal covers the same config shape without
|
||||
import yaml # type: ignore[import-not-found] # optional dep; fall back silently if absent
|
||||
with open(path, encoding="utf-8") as f:
|
||||
import yaml # type: ignore[import-not-found]
|
||||
with open(path) as f:
|
||||
return yaml.safe_load(f)
|
||||
except ImportError:
|
||||
return _load_config_minimal(path)
|
||||
@@ -654,19 +461,13 @@ def _load_config_minimal(path: str) -> dict[str, Any]:
|
||||
item map: scalars + lists of scalars. Does NOT support nested lists,
|
||||
YAML anchors, multi-doc, or flow style.
|
||||
"""
|
||||
with open(path, encoding="utf-8") as f:
|
||||
with open(path) as f:
|
||||
lines = f.readlines()
|
||||
return _parse_minimal_yaml(lines)
|
||||
|
||||
|
||||
def _parse_minimal_yaml(lines: list[str]) -> dict[str, Any]:
|
||||
"""Hand-rolled subset parser. See _load_config_minimal docstring.
|
||||
|
||||
C901: function is necessarily long — it implements a finite-state YAML
|
||||
subset (scalars, maps, lists of maps at fixed depth). No utility refactors
|
||||
meaningfully reduce length without degrading readability. All branches
|
||||
are exhaustively tested in test_parse_minimal_yaml.py.
|
||||
"""
|
||||
def _parse_minimal_yaml(lines: list[str]) -> dict[str, Any]: # noqa: C901
|
||||
"""Hand-rolled subset parser. See _load_config_minimal docstring."""
|
||||
# Strip comments + blank lines but preserve indentation.
|
||||
cleaned: list[tuple[int, str]] = []
|
||||
for raw in lines:
|
||||
@@ -819,8 +620,8 @@ def render_status(
|
||||
|
||||
state is "success" if every item has at least one valid ack
|
||||
(body section presence is informational only — peer-ack is the
|
||||
real gate). tier:low PRs receive state="success" (soft-fail — no
|
||||
acks required); the description carries "[info tier:low]" prefix.
|
||||
real gate). "pending" is reserved for the soft-fail path
|
||||
(tier:low) and is set by the caller.
|
||||
"""
|
||||
n = len(items)
|
||||
fully_acked = [
|
||||
@@ -839,18 +640,15 @@ def render_status(
|
||||
shown += f", +{len(missing) - 3}"
|
||||
desc_parts.append(f"missing: {shown}")
|
||||
if missing_body:
|
||||
shown = ", ".join(missing_body[:3])
|
||||
if len(missing_body) > 3:
|
||||
shown += f", +{len(missing_body) - 3}"
|
||||
desc_parts.append(f"body-unfilled: {shown}")
|
||||
state = "success" if not missing and not missing_body else "failure"
|
||||
desc_parts.append(f"body-unfilled: {len(missing_body)}")
|
||||
state = "success" if not missing else "failure"
|
||||
return state, " — ".join(desc_parts)
|
||||
|
||||
|
||||
def get_tier_mode(pr: dict[str, Any], cfg: dict[str, Any]) -> str:
|
||||
"""Read tier label, return 'hard' or 'soft' per cfg.tier_failure_mode."""
|
||||
labels = pr.get("labels") or []
|
||||
tier_labels = [label.get("name", "") for label in labels if (label.get("name", "") or "").startswith("tier:")]
|
||||
tier_labels = [l.get("name", "") for l in labels if (l.get("name", "") or "").startswith("tier:")]
|
||||
mode_map = cfg.get("tier_failure_mode") or {}
|
||||
default_mode = cfg.get("default_mode", "hard")
|
||||
for tl in tier_labels:
|
||||
@@ -859,83 +657,6 @@ def get_tier_mode(pr: dict[str, Any], cfg: dict[str, Any]) -> str:
|
||||
return default_mode
|
||||
|
||||
|
||||
def is_high_risk(pr: dict[str, Any], cfg: dict[str, Any]) -> bool:
|
||||
"""Return True when the PR is high-risk per RFC#450 Option C.
|
||||
|
||||
A PR is high-risk when ANY of:
|
||||
- it carries the `tier:high` label (mechanically strictest tier), or
|
||||
- it carries any label listed in cfg.high_risk_labels.
|
||||
|
||||
High-risk PRs use `required_teams_high_risk` (when set on an item)
|
||||
instead of the default `required_teams`. Items without
|
||||
`required_teams_high_risk` are unaffected (the default applies).
|
||||
|
||||
Governance fix for internal#442 — closes the inconsistency between
|
||||
sop-tier-check (tier-aware) and sop-checklist (was tier-blind).
|
||||
"""
|
||||
label_set = {(label.get("name") or "") for label in (pr.get("labels") or [])}
|
||||
if "tier:high" in label_set:
|
||||
return True
|
||||
high_risk_labels = set(cfg.get("high_risk_labels") or [])
|
||||
return bool(label_set & high_risk_labels)
|
||||
|
||||
|
||||
def resolve_required_teams(item: dict[str, Any], high_risk: bool) -> list[str]:
|
||||
"""Pick the active required_teams list for an item.
|
||||
|
||||
When high_risk is True AND the item declares a non-empty
|
||||
`required_teams_high_risk`, return that. Else fall back to
|
||||
`required_teams`. Keeping this in one helper means the gate's
|
||||
decision shape stays single-sited even as items grow.
|
||||
"""
|
||||
if high_risk:
|
||||
elevated = item.get("required_teams_high_risk") or []
|
||||
if elevated:
|
||||
return list(elevated)
|
||||
return list(item.get("required_teams") or [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CI status validation for testing-class AI acks (internal#760 CTO hardening)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Slugs that require CI / all-required green before an AI ack is valid.
|
||||
_TESTING_CLASS_SLUGS = {"comprehensive-testing", "local-postgres-e2e", "staging-smoke"}
|
||||
|
||||
# Human-only carve-out: these items can NEVER be acked by AI, regardless
|
||||
# of config drift. Any item in this set MUST NOT have ai_ack_eligible.
|
||||
# migration / schema are future-proofing — not yet in config items, but
|
||||
# the code guard rejects them proactively (CTO hardening, msg 1388c76f).
|
||||
_HUMAN_ONLY_SLUGS = {"root-cause", "no-backwards-compat", "migration", "schema"}
|
||||
|
||||
|
||||
def get_ci_status(client: GiteaClient, owner: str, repo: str, sha: str) -> str:
|
||||
"""Return the state of CI / all-required (pull_request) for `sha`.
|
||||
|
||||
Looks through the commit statuses and returns the state string
|
||||
("success", "failure", "pending", "error") or "missing" if the
|
||||
context is not found. This prevents an AI agent from attesting
|
||||
"tests pass" independently of the actual CI run.
|
||||
"""
|
||||
code, data = client._req( # noqa: SLF001
|
||||
"GET", f"/repos/{owner}/{repo}/statuses/{sha}"
|
||||
)
|
||||
if code != 200:
|
||||
return "unknown"
|
||||
if not data or not isinstance(data, list):
|
||||
return "missing"
|
||||
# Gitea returns statuses newest-first. Find the latest for our context.
|
||||
for status in data:
|
||||
if status.get("context") == "CI / all-required (pull_request)":
|
||||
return status.get("state", "unknown")
|
||||
return "missing"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--owner", required=True)
|
||||
@@ -961,17 +682,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"thing BP sees is the POSTed status. Useful for local debugging."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--max-comments",
|
||||
type=int,
|
||||
default=int(os.environ.get("SOP_CHECKLIST_MAX_COMMENTS") or 5000),
|
||||
help=(
|
||||
"Hard cap on comments fetched from the PR. Above this we post "
|
||||
"a SOFT-pending status with a 'skipping due to volume' note "
|
||||
"instead of OOM'ing the runner (task #369). Override with the "
|
||||
"SOP_CHECKLIST_MAX_COMMENTS env var. Set 0 to disable the cap."
|
||||
),
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
token = os.environ.get("GITEA_TOKEN", "")
|
||||
@@ -982,7 +692,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
cfg = load_config(args.config)
|
||||
items: list[dict[str, Any]] = cfg["items"]
|
||||
items_by_slug = {it["slug"]: it for it in items}
|
||||
na_gates: dict[str, Any] = cfg.get("n/a_gates", {})
|
||||
numeric_aliases = {
|
||||
int(it["numeric_alias"]): it["slug"] for it in items if it.get("numeric_alias")
|
||||
}
|
||||
@@ -1005,62 +714,16 @@ def main(argv: list[str] | None = None) -> int:
|
||||
print("::error::PR payload missing user.login or head.sha", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
max_comments_cap = args.max_comments if args.max_comments and args.max_comments > 0 else None
|
||||
comments = client.get_issue_comments(
|
||||
args.owner, args.repo, args.pr, max_comments=max_comments_cap
|
||||
)
|
||||
|
||||
# Volume short-circuit: PRs with thousands of bot-relay comments
|
||||
# (the mc#1242-class OOM source) get a soft 'volume-skipped' status
|
||||
# so the gate doesn't churn the runner; reviewers can re-trigger by
|
||||
# editing the PR or filing a fresh PR with the housekeeping comments
|
||||
# split off. Cap-hit means we couldn't see the WHOLE history, so we
|
||||
# can't fairly post failure — pending is the safe default.
|
||||
volume_skipped = bool(max_comments_cap and len(comments) >= max_comments_cap)
|
||||
|
||||
# High-risk classification (RFC#450 Option C, governance fix for
|
||||
# internal#442). Computed ONCE per PR — used by both the probe
|
||||
# closure and compute_ack_state so the elevation decision is
|
||||
# single-sited.
|
||||
high_risk = is_high_risk(pr, cfg)
|
||||
comments = client.get_issue_comments(args.owner, args.repo, args.pr)
|
||||
|
||||
# Build team-membership probe closure that caches results per
|
||||
# (user, team-id) so a user acking multiple items only triggers
|
||||
# one membership lookup per team.
|
||||
team_member_cache: dict[tuple[str, int], bool | None] = {}
|
||||
|
||||
# Pre-resolve the ai-sop-ack team id once (None if the team does not exist).
|
||||
ai_sop_ack_team_id = client.resolve_team_id(args.owner, "ai-sop-ack")
|
||||
|
||||
def probe(slug: str, users: list[str]) -> list[str]:
|
||||
# `slug` may be either an items-key (compute_ack_state caller) OR
|
||||
# an n/a-gate key (compute_na_state caller). Previously this hard
|
||||
# KeyError'd on the n/a-gate path when slug was e.g. "security-review"
|
||||
# — that's a config gate, not an item — so the gate would crash
|
||||
# instead of falling back to the gate's own required_teams. Fix
|
||||
# task #369 follow-up to issue #355.
|
||||
if slug in items_by_slug:
|
||||
item = items_by_slug[slug]
|
||||
team_names: list[str] = resolve_required_teams(item, high_risk)
|
||||
elif slug in na_gates:
|
||||
# n/a-gate configs carry `required_teams` directly (see
|
||||
# sop-checklist-config.yaml: n/a_gates.<gate>.required_teams).
|
||||
gate_cfg = na_gates[slug] or {}
|
||||
team_names = list(gate_cfg.get("required_teams") or [])
|
||||
if not team_names:
|
||||
print(
|
||||
f"::warning::n/a-gate '{slug}' has no required_teams; "
|
||||
"fail-closed (no users will be approved)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
# Unknown slug — fail closed, log so we can find config drift.
|
||||
print(
|
||||
f"::warning::probe() called with slug '{slug}' which is "
|
||||
f"neither an items entry nor an n/a-gate; fail-closed",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return []
|
||||
item = items_by_slug[slug]
|
||||
team_names: list[str] = item["required_teams"]
|
||||
# Resolve names → ids. NOTE: orgs/{org}/teams/search may not be
|
||||
# available — fall back to the list endpoint.
|
||||
team_ids: list[int] = []
|
||||
@@ -1068,14 +731,14 @@ def main(argv: list[str] | None = None) -> int:
|
||||
tid = client.resolve_team_id(args.owner, tn)
|
||||
if tid is None:
|
||||
# Try the list endpoint as a fallback.
|
||||
code, data = client._req( # noqa: SLF001 # internal helper; called from loop in caller context
|
||||
code, data = client._req( # noqa: SLF001
|
||||
"GET", f"/orgs/{args.owner}/teams"
|
||||
)
|
||||
if code == 200 and isinstance(data, list):
|
||||
for t in data:
|
||||
if t.get("name") == tn:
|
||||
tid = t.get("id")
|
||||
client._team_id_cache[(args.owner, tn)] = tid # noqa: SLF001 # write-through cache; intentional side-effect for reuse across calls
|
||||
client._team_id_cache[(args.owner, tn)] = tid # noqa: SLF001
|
||||
break
|
||||
if tid is not None:
|
||||
team_ids.append(tid)
|
||||
@@ -1086,18 +749,14 @@ def main(argv: list[str] | None = None) -> int:
|
||||
file=sys.stderr,
|
||||
)
|
||||
approved: list[str] = []
|
||||
rejected_ai_ineligible: list[str] = []
|
||||
rejected_ci_not_green: list[str] = []
|
||||
for u in users:
|
||||
# 1) Human required_teams membership check
|
||||
in_human_team = False
|
||||
for tid in team_ids:
|
||||
cache_key = (u, tid)
|
||||
if cache_key not in team_member_cache:
|
||||
team_member_cache[cache_key] = client.is_team_member(tid, u)
|
||||
result = team_member_cache[cache_key]
|
||||
if result is True:
|
||||
in_human_team = True
|
||||
approved.append(u)
|
||||
break
|
||||
if result is None:
|
||||
print(
|
||||
@@ -1107,74 +766,19 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
# Treat as not-in-team for this user/team pair; loop
|
||||
# may still find membership in another team.
|
||||
if in_human_team:
|
||||
approved.append(u)
|
||||
continue
|
||||
|
||||
# 2) AI-sop-ack team membership check (only for items that allow it).
|
||||
if slug in items_by_slug:
|
||||
item = items_by_slug[slug]
|
||||
# Defensive: human-only carve-out is enforced in code, not just
|
||||
# config. Even if ai_ack_eligible were mistakenly added to a
|
||||
# migration/schema item, the AI path is rejected here.
|
||||
if slug in _HUMAN_ONLY_SLUGS:
|
||||
rejected_ai_ineligible.append(u)
|
||||
continue
|
||||
if item.get("ai_ack_eligible") and ai_sop_ack_team_id is not None:
|
||||
cache_key = (u, ai_sop_ack_team_id)
|
||||
if cache_key not in team_member_cache:
|
||||
team_member_cache[cache_key] = client.is_team_member(
|
||||
ai_sop_ack_team_id, u
|
||||
)
|
||||
result = team_member_cache[cache_key]
|
||||
if result is True:
|
||||
# 2a) Testing-class items require real CI artifact evidence.
|
||||
if slug in _TESTING_CLASS_SLUGS:
|
||||
ci_state = get_ci_status(
|
||||
client, args.owner, args.repo, head_sha
|
||||
)
|
||||
if ci_state != "success":
|
||||
print(
|
||||
f"::warning::AI ack for {slug} rejected: "
|
||||
f"CI / all-required is {ci_state}, not success",
|
||||
file=sys.stderr,
|
||||
)
|
||||
rejected_ci_not_green.append(u)
|
||||
continue
|
||||
approved.append(u)
|
||||
continue
|
||||
# If we get here, user is not approved for this slug.
|
||||
rejected_ai_ineligible.append(u)
|
||||
return approved
|
||||
|
||||
ack_state = compute_ack_state(
|
||||
comments, author, items_by_slug, numeric_aliases, probe, high_risk=high_risk
|
||||
)
|
||||
ack_state = compute_ack_state(comments, author, items_by_slug, numeric_aliases, probe)
|
||||
body_state = {it["slug"]: section_marker_present(body, it["pr_section_marker"]) for it in items}
|
||||
|
||||
state, description = render_status(items, ack_state, body_state)
|
||||
mode = get_tier_mode(pr, cfg)
|
||||
if mode == "soft":
|
||||
# tier:low: acks are informational only — post success so BP gate passes.
|
||||
# Description carries "[info tier:low]" prefix so reviewers know acks
|
||||
# were not required (vs a tier:medium+ PR that truly passed all acks).
|
||||
state = "success"
|
||||
description = f"[info tier:low] {description}"
|
||||
if volume_skipped:
|
||||
# Above the comment-cap — we may have a partial view. Soft-pend
|
||||
# so neither BP nor the author gets stuck; surface the cap so
|
||||
# reviewers know what's up. No-block at the gate level.
|
||||
if state == "failure" and mode == "soft":
|
||||
state = "pending"
|
||||
description = (
|
||||
f"[volume-skipped] comment-cap={max_comments_cap} hit; please file "
|
||||
f"a fresh PR with bot-relay history split off (#369). {description}"
|
||||
)
|
||||
description = f"[soft-fail tier:low] {description}"
|
||||
|
||||
# Diagnostics to job log.
|
||||
print(
|
||||
f"::notice::PR #{args.pr} author={author} head={head_sha[:7]} "
|
||||
f"mode={mode} risk_class={'high' if high_risk else 'default'}"
|
||||
)
|
||||
print(f"::notice::PR #{args.pr} author={author} head={head_sha[:7]} mode={mode}")
|
||||
for it in items:
|
||||
slug = it["slug"]
|
||||
ackers = ack_state[slug]["ackers"]
|
||||
@@ -1205,46 +809,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
description=description, target_url=target_url,
|
||||
)
|
||||
print(f"::notice::status posted: {args.status_context} → {state}")
|
||||
|
||||
# --- N/A gate status (RFC#324 §N/A follow-up) ---
|
||||
# Post a separate status so review-check.sh can discover N/A declarations
|
||||
# and waive the Gitea-approve requirement for that gate.
|
||||
na_state: dict[str, dict[str, Any]] = {}
|
||||
if na_gates:
|
||||
na_state = compute_na_state(comments, author, na_gates, probe)
|
||||
|
||||
na_descs: list[str] = []
|
||||
for gate, s in na_state.items():
|
||||
if s["declared"]:
|
||||
na_descs.append(gate)
|
||||
decl = s["decl_ackers"]
|
||||
rej = s["rejected"]["not_in_team"]
|
||||
if decl:
|
||||
print(f"::notice:: [N/A OK] {gate} — declared by {','.join(decl)}")
|
||||
if rej:
|
||||
print(
|
||||
f"::notice:: [N/A REJ] {gate} — not-in-team: {','.join(rej)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
na_desc = ", ".join(sorted(na_descs)) if na_descs else "(none)"
|
||||
na_status_state = "success" if na_descs else "pending"
|
||||
# review-check.sh reads the description to discover which gates are N/A.
|
||||
# Include the gate names so it can grep for them.
|
||||
na_description = f"N/A: {na_desc}" if na_descs else "N/A: (none)"
|
||||
|
||||
if not args.dry_run:
|
||||
client.post_status(
|
||||
args.owner, args.repo, head_sha,
|
||||
state=na_status_state,
|
||||
context="sop-checklist / na-declarations (pull_request)",
|
||||
description=na_description,
|
||||
target_url=target_url,
|
||||
)
|
||||
print(
|
||||
f"::notice::na-declarations status → {na_status_state}: {na_description}"
|
||||
)
|
||||
|
||||
# By default exit 0 — the POSTed status IS the gate, NOT the job
|
||||
# conclusion. If the job exits 1 BP will see TWO failure signals
|
||||
# (one from the job's auto-status, one from our POST), making the
|
||||
@@ -104,13 +104,10 @@ if [ "${SOP_REFIRE_DISABLE_RATE_LIMIT:-}" != "1" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects.
|
||||
# The canonical workflow intentionally fail-opens the job conclusion
|
||||
# (`bash .gitea/scripts/sop-tier-check.sh || true`) while Gitea branch
|
||||
# protection enforces reviewer approvals separately. Keep the refire path
|
||||
# aligned with that workflow status behavior; otherwise /refire-tier-check can
|
||||
# post a hard failure that the canonical pull_request_target workflow would
|
||||
# not publish.
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects. Capture exit code.
|
||||
# The canonical script reads tier label, walks approving reviewers, and
|
||||
# evaluates the AND-composition expression — we want the SAME gate, not
|
||||
# a different gate.
|
||||
#
|
||||
# SOP_REFIRE_TIER_CHECK_SCRIPT env var lets tests substitute a mock —
|
||||
# sop-tier-check.sh uses bash 4+ associative arrays which trigger a known
|
||||
@@ -126,6 +123,7 @@ fi
|
||||
|
||||
# Re-invoke. Pipe stdout/stderr through so the runner log shows the
|
||||
# tier-check decision inline.
|
||||
set +e
|
||||
GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
GITEA_HOST="$GITEA_HOST" \
|
||||
REPO="$REPO" \
|
||||
@@ -133,8 +131,9 @@ GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
PR_AUTHOR="$PR_AUTHOR" \
|
||||
SOP_DEBUG="${SOP_DEBUG:-0}" \
|
||||
SOP_LEGACY_CHECK="${SOP_LEGACY_CHECK:-0}" \
|
||||
bash "$SCRIPT" || true
|
||||
TIER_EXIT=0
|
||||
bash "$SCRIPT"
|
||||
TIER_EXIT=$?
|
||||
set -e
|
||||
debug "sop-tier-check.sh exit=$TIER_EXIT"
|
||||
|
||||
# 4. POST the resulting status.
|
||||
|
||||
+29
-156
@@ -47,9 +47,7 @@ What this script does, per `.gitea/workflows/status-reaper.yml` invocation:
|
||||
Parse context as `<workflow_name> / <job_name> (push)`.
|
||||
Look up workflow_name in the trigger map:
|
||||
- missing → log ::notice:: and skip (conservative).
|
||||
- has_push_trigger=True and description == "Has been cancelled"
|
||||
→ compensate cancelled/superseded push noise.
|
||||
- has_push_trigger=True otherwise → preserve (real defect signal).
|
||||
- has_push_trigger=True → preserve (real defect signal).
|
||||
- has_push_trigger=False → POST a compensating
|
||||
`state=success` status to /statuses/{sha} with the same
|
||||
context (Gitea de-dups by context) and a description
|
||||
@@ -60,10 +58,9 @@ What this script does, per `.gitea/workflows/status-reaper.yml` invocation:
|
||||
even if another tick happens before the runner finishes.
|
||||
|
||||
What it does NOT do:
|
||||
- Touch ` (pull_request)` contexts unless the exact same
|
||||
workflow/job has a successful ` (push)` context on the same
|
||||
default-branch SHA. That case is post-merge status pollution, not
|
||||
an unproven PR gate.
|
||||
- Touch any context NOT ending in ` (push)`. The required-checks on
|
||||
main (verified 2026-05-11) all have ` (pull_request)` suffixes;
|
||||
they CANNOT be reached by this code path.
|
||||
- Compensate `error`/`pending` states. Only `failure` — the only one
|
||||
Gitea emits for the hardcoded-suffix bug.
|
||||
- Write to non-default branches. WATCH_BRANCH is sourced from
|
||||
@@ -94,9 +91,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -123,36 +118,19 @@ WORKFLOWS_DIR = _env("WORKFLOWS_DIR", default=".gitea/workflows")
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
API_TIMEOUT_SEC = int(_env("STATUS_REAPER_API_TIMEOUT_SEC", default="30") or "30")
|
||||
API_RETRIES = int(_env("STATUS_REAPER_API_RETRIES", default="3") or "3")
|
||||
API_RETRY_SLEEP_SEC = float(_env("STATUS_REAPER_API_RETRY_SLEEP_SEC", default="2") or "2")
|
||||
|
||||
# Compensating-status description prefix. Used as the marker so a human
|
||||
# auditing commit statuses can tell at a glance that the green was
|
||||
# synthetic, not a real CI pass. Kept stable; downstream tooling
|
||||
# (e.g. main-red-watchdog visual diff) MAY key on it.
|
||||
PUSH_COMPENSATION_DESCRIPTION = (
|
||||
COMPENSATION_DESCRIPTION = (
|
||||
"Compensated by status-reaper (workflow has no push: trigger; "
|
||||
"Gitea 1.22.6 hardcoded-suffix bug — see .gitea/scripts/status-reaper.py)"
|
||||
)
|
||||
# Backward-compatible alias for older tests/tooling that predate the split
|
||||
# between push-suffix compensation and pull-request-shadow compensation.
|
||||
COMPENSATION_DESCRIPTION = PUSH_COMPENSATION_DESCRIPTION
|
||||
PR_SHADOW_COMPENSATION_DESCRIPTION = (
|
||||
"Compensated by status-reaper (default-branch pull_request status "
|
||||
"shadowed by successful push status on same SHA; see "
|
||||
".gitea/scripts/status-reaper.py)"
|
||||
)
|
||||
CANCELLED_PUSH_COMPENSATION_DESCRIPTION = (
|
||||
"Compensated by status-reaper (push run was cancelled/superseded; "
|
||||
"Gitea 1.22.6 reports cancelled runs as failure statuses)"
|
||||
)
|
||||
CANCELLED_DESCRIPTION = "Has been cancelled"
|
||||
|
||||
# Context suffix the reaper acts on. Gitea hardcodes this for ALL
|
||||
# default-branch workflow runs.
|
||||
PUSH_SUFFIX = " (push)"
|
||||
PULL_REQUEST_SUFFIX = " (pull_request)"
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
@@ -204,27 +182,13 @@ def api(
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
attempts = max(API_RETRIES, 1)
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=API_TIMEOUT_SEC) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
break
|
||||
except (TimeoutError, socket.timeout, urllib.error.URLError, OSError) as e:
|
||||
if attempt >= attempts:
|
||||
raise ApiError(
|
||||
f"{method} {path} failed after {attempts} attempts: {e}"
|
||||
) from e
|
||||
print(
|
||||
f"::warning::{method} {path} transient API error "
|
||||
f"(attempt {attempt}/{attempts}): {e}; retrying"
|
||||
)
|
||||
time.sleep(API_RETRY_SLEEP_SEC)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
@@ -393,38 +357,24 @@ def get_combined_status(sha: str) -> dict:
|
||||
# --------------------------------------------------------------------------
|
||||
# Context parsing
|
||||
# --------------------------------------------------------------------------
|
||||
def parse_suffixed_context(context: str, suffix: str) -> tuple[str, str] | None:
|
||||
"""Parse `<workflow_name> / <job_name> (<event>)` into
|
||||
def parse_push_context(context: str) -> tuple[str, str] | None:
|
||||
"""Parse `<workflow_name> / <job_name> (push)` into
|
||||
(workflow_name, job_name).
|
||||
|
||||
Returns None if the context doesn't match the shape (caller skips).
|
||||
Strict: requires the trailing suffix and at least one ` / `
|
||||
Strict: requires the trailing ` (push)` and at least one ` / `
|
||||
separator. Anything else is left alone.
|
||||
"""
|
||||
if not context.endswith(suffix):
|
||||
if not context.endswith(PUSH_SUFFIX):
|
||||
return None
|
||||
head = context[: -len(suffix)]
|
||||
head = context[: -len(PUSH_SUFFIX)] # strip " (push)"
|
||||
if " / " not in head:
|
||||
# No workflow/job separator — not the bug shape we compensate.
|
||||
return None
|
||||
workflow_name, job_name = head.split(" / ", 1)
|
||||
return workflow_name, job_name
|
||||
|
||||
|
||||
def parse_push_context(context: str) -> tuple[str, str] | None:
|
||||
"""Parse `<workflow_name> / <job_name> (push)` into
|
||||
(workflow_name, job_name)."""
|
||||
return parse_suffixed_context(context, PUSH_SUFFIX)
|
||||
|
||||
|
||||
def push_equivalent_context(context: str) -> str | None:
|
||||
"""Return the matching `(push)` context for a `(pull_request)` context."""
|
||||
parsed = parse_suffixed_context(context, PULL_REQUEST_SUFFIX)
|
||||
if parsed is None:
|
||||
return None
|
||||
workflow_name, job_name = parsed
|
||||
return f"{workflow_name} / {job_name}{PUSH_SUFFIX}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Compensating POST
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -433,7 +383,6 @@ def post_compensating_status(
|
||||
context: str,
|
||||
target_url: str | None,
|
||||
*,
|
||||
description: str = PUSH_COMPENSATION_DESCRIPTION,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""POST a `state=success` to /repos/{o}/{r}/statuses/{sha} with the
|
||||
@@ -445,7 +394,7 @@ def post_compensating_status(
|
||||
payload: dict[str, Any] = {
|
||||
"context": context,
|
||||
"state": "success",
|
||||
"description": description,
|
||||
"description": COMPENSATION_DESCRIPTION,
|
||||
}
|
||||
# Echo the original target_url when present so a human auditing
|
||||
# the (now-green) compensated status can still reach the run logs
|
||||
@@ -482,8 +431,7 @@ def reap(
|
||||
Returns counters for observability:
|
||||
{compensated, preserved_real_push, preserved_unknown,
|
||||
preserved_non_failure, preserved_non_push_suffix,
|
||||
preserved_unparseable, compensated_pr_shadowed_by_push_success,
|
||||
preserved_pr_without_push_success, compensated_cancelled_push,
|
||||
preserved_unparseable,
|
||||
compensated_contexts: [<context>, ...]}
|
||||
|
||||
`compensated_contexts` is rev2-added so `reap_branch` can build
|
||||
@@ -496,18 +444,10 @@ def reap(
|
||||
"preserved_non_failure": 0,
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_contexts": [],
|
||||
}
|
||||
|
||||
statuses = combined.get("statuses") or []
|
||||
successful_contexts = {
|
||||
(s.get("context") or "")
|
||||
for s in statuses
|
||||
if isinstance(s, dict) and (s.get("status") or s.get("state") or "") == "success"
|
||||
}
|
||||
for s in statuses:
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
@@ -531,31 +471,9 @@ def reap(
|
||||
counters["preserved_non_failure"] += 1
|
||||
continue
|
||||
|
||||
# Default-branch `pull_request` contexts can be stale shadows of
|
||||
# the exact same workflow/job already proven by the successful
|
||||
# `push` context on the same SHA. Compensate only that narrow
|
||||
# shape; a missing or failed push equivalent remains a real gate
|
||||
# signal and is preserved.
|
||||
push_equivalent = push_equivalent_context(context)
|
||||
if push_equivalent is not None:
|
||||
if push_equivalent in successful_contexts:
|
||||
post_compensating_status(
|
||||
sha,
|
||||
context,
|
||||
s.get("target_url"),
|
||||
description=PR_SHADOW_COMPENSATION_DESCRIPTION,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
counters["compensated"] += 1
|
||||
counters["compensated_pr_shadowed_by_push_success"] += 1
|
||||
counters["compensated_contexts"].append(context)
|
||||
else:
|
||||
counters["preserved_pr_without_push_success"] += 1
|
||||
continue
|
||||
|
||||
# Only `(push)`-suffix contexts hit the hardcoded-suffix bug.
|
||||
# Other failed contexts are preserved unless handled by the
|
||||
# pull-request-shadow rule above.
|
||||
# Branch-protection required checks (e.g. `Secret scan / Scan
|
||||
# diff (pull_request)`) are NOT reachable from this path.
|
||||
if not context.endswith(PUSH_SUFFIX):
|
||||
counters["preserved_non_push_suffix"] += 1
|
||||
continue
|
||||
@@ -575,27 +493,8 @@ def reap(
|
||||
counters["preserved_unknown"] += 1
|
||||
continue
|
||||
|
||||
if (s.get("description") or "").strip() == CANCELLED_DESCRIPTION:
|
||||
# Gitea 1.22.6 maps cancelled action runs to failure commit
|
||||
# statuses. During merge bursts, older push runs can be
|
||||
# superseded and cancelled even though a newer run for the
|
||||
# same branch is the real signal. Compensate only the exact
|
||||
# Gitea cancellation description; real push failures remain red.
|
||||
post_compensating_status(
|
||||
sha,
|
||||
context,
|
||||
s.get("target_url"),
|
||||
description=CANCELLED_PUSH_COMPENSATION_DESCRIPTION,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
counters["compensated"] += 1
|
||||
counters["compensated_cancelled_push"] += 1
|
||||
counters["compensated_contexts"].append(context)
|
||||
continue
|
||||
|
||||
if workflow_trigger_map[workflow_name]:
|
||||
# Real push trigger with a non-cancelled failure description
|
||||
# remains a defect signal. Preserve.
|
||||
# Real push trigger → real defect signal. Preserve.
|
||||
counters["preserved_real_push"] += 1
|
||||
continue
|
||||
|
||||
@@ -641,10 +540,11 @@ def list_recent_commit_shas(branch: str, limit: int) -> list[str]:
|
||||
(verified via vendor-truth probe 2026-05-11 against
|
||||
git.moleculesai.app — `feedback_smoke_test_vendor_truth_not_shape_match`).
|
||||
|
||||
Raises ApiError on non-2xx OR on unexpected response shape. The
|
||||
branch-level caller soft-skips this tick because the next scheduled
|
||||
tick can safely retry the listing. Per-SHA status/write errors remain
|
||||
separate and must not be mislabeled as commit-list outages.
|
||||
Raises ApiError on non-2xx OR on unexpected response shape. This is
|
||||
a HARD halt — without the commit list the sweep can't proceed. (The
|
||||
per-SHA error isolation downstream is a different concern: tolerating
|
||||
a transient 5xx on ONE commit's status is best-effort; losing the
|
||||
commit list itself means we don't even know which commits to try.)
|
||||
"""
|
||||
_, body = api(
|
||||
"GET",
|
||||
@@ -685,28 +585,7 @@ def reap_branch(
|
||||
- compensated_per_sha: {<sha_full>: [<context>, ...]} — only
|
||||
SHAs that actually got at least one compensation are included
|
||||
"""
|
||||
try:
|
||||
shas = list_recent_commit_shas(branch, limit)
|
||||
except ApiError as e:
|
||||
print(
|
||||
"::warning::status-reaper skipped this tick because the "
|
||||
f"commit list could not be read after retries: {e}"
|
||||
)
|
||||
return {
|
||||
"scanned_shas": 0,
|
||||
"compensated": 0,
|
||||
"preserved_real_push": 0,
|
||||
"preserved_unknown": 0,
|
||||
"preserved_non_failure": 0,
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_per_sha": {},
|
||||
"skipped": True,
|
||||
"skip_reason": "commit-list-api-error",
|
||||
}
|
||||
shas = list_recent_commit_shas(branch, limit)
|
||||
|
||||
aggregate: dict[str, Any] = {
|
||||
"scanned_shas": 0,
|
||||
@@ -716,9 +595,6 @@ def reap_branch(
|
||||
"preserved_non_failure": 0,
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_per_sha": {},
|
||||
}
|
||||
|
||||
@@ -756,9 +632,6 @@ def reap_branch(
|
||||
"preserved_non_failure",
|
||||
"preserved_non_push_suffix",
|
||||
"preserved_unparseable",
|
||||
"compensated_pr_shadowed_by_push_success",
|
||||
"compensated_cancelled_push",
|
||||
"preserved_pr_without_push_success",
|
||||
):
|
||||
aggregate[key] += per_sha[key]
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ def scenario() -> str:
|
||||
p = os.path.join(STATE_DIR, "scenario")
|
||||
if not os.path.isfile(p):
|
||||
return "T1_success"
|
||||
with open(p, encoding="utf-8") as f:
|
||||
with open(p) as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
|
||||
@@ -16,12 +16,6 @@ Scenarios:
|
||||
T7_team_member — team membership → 204 (member) → exit 0
|
||||
T8_team_not_member — team membership → 404 (not a member) → exit 1
|
||||
T9_team_403 — team membership → 403 (token not in team) → exit 1
|
||||
T14_non_default_base — open PR targeting staging → script exits 0 (no-op)
|
||||
T15_comments_agent_approval — reviews empty; comments have "[core-qa-agent] APPROVED" → exit 0
|
||||
T16_comments_generic_approval — reviews empty; comments have "APPROVED" by team member → exit 0
|
||||
T17_comments_no_approval — reviews empty; comments have no approval keywords → exit 1
|
||||
T18_review_wrong_team_comment_right_team — review candidate 404s, comment candidate passes
|
||||
T19_ai_sop_ack_approved — ai-sop-ack member APPROVED review → team probe 404 → exit 1
|
||||
|
||||
Usage:
|
||||
FIXTURE_STATE_DIR=/tmp/x python3 _review_check_fixture.py 8080
|
||||
@@ -34,6 +28,7 @@ import re
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
|
||||
STATE_DIR = os.environ.get("FIXTURE_STATE_DIR", "/tmp")
|
||||
|
||||
|
||||
@@ -41,7 +36,7 @@ def scenario() -> str:
|
||||
p = os.path.join(STATE_DIR, "scenario")
|
||||
if not os.path.isfile(p):
|
||||
return "T1_pr_open"
|
||||
with open(p, encoding="utf-8") as f:
|
||||
with open(p) as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
@@ -81,29 +76,25 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
# GET /repos/{owner}/{name}/pulls/{pr_number}
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/pulls/(\d+)$", path)
|
||||
if m:
|
||||
pr_num = m.group(3)
|
||||
owner, name, pr_num = m.group(1), m.group(2), m.group(3)
|
||||
if sc == "T2_pr_closed":
|
||||
return self._json(200, {
|
||||
"number": int(pr_num),
|
||||
"state": "closed",
|
||||
"head": {"sha": "deadbeef0000111122223333444455556666"},
|
||||
"base": {"ref": "main"},
|
||||
"user": {"login": "alice"},
|
||||
})
|
||||
return self._json(200, {
|
||||
"number": int(pr_num),
|
||||
"state": "open",
|
||||
"head": {"sha": "deadbeef0000111122223333444455556666"},
|
||||
"base": {"ref": "staging" if sc == "T14_non_default_base" else "main"},
|
||||
"user": {"login": "alice"},
|
||||
})
|
||||
|
||||
# GET /repos/{owner}/{name}/pulls/{pr_number}/reviews
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/pulls/(\d+)/reviews$", path)
|
||||
if m:
|
||||
if sc in ("T4_reviews_empty", "T5_reviews_only_author",
|
||||
"T15_comments_agent_approval", "T16_comments_generic_approval",
|
||||
"T17_comments_no_approval"):
|
||||
if sc in ("T4_reviews_empty", "T5_reviews_only_author"):
|
||||
return self._json(200, [])
|
||||
if sc == "T6_reviews_dismissed":
|
||||
return self._json(200, [{
|
||||
@@ -117,65 +108,22 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
{"state": "CHANGES_REQUESTED", "dismissed": False, "user": {"login": "bob"}, "commit_id": "abc1234"},
|
||||
{"state": "APPROVED", "dismissed": False, "user": {"login": "core-devops"}, "commit_id": "abc1234"},
|
||||
])
|
||||
if sc == "T19_ai_sop_ack_approved":
|
||||
# ai-sop-ack member submitted APPROVED review — must NOT count
|
||||
# toward qa-review (team_id=20) or security-review (team_id=21).
|
||||
return self._json(200, [
|
||||
{"state": "APPROVED", "dismissed": False, "user": {"login": "ai-reviewer"}, "commit_id": "abc1234"},
|
||||
])
|
||||
# Default: one non-author APPROVED
|
||||
return self._json(200, [
|
||||
{"state": "APPROVED", "dismissed": False, "user": {"login": "core-devops"}, "commit_id": "abc1234"},
|
||||
])
|
||||
|
||||
# GET /repos/{owner}/{name}/issues/{pr_number}/comments
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/issues/(\d+)/comments$", path)
|
||||
if m:
|
||||
if sc == "T15_comments_agent_approval":
|
||||
return self._json(200, [
|
||||
{"user": {"login": "core-qa-agent"}, "body": "[core-qa-agent] APPROVED this PR. Good changes.", "id": 1},
|
||||
{"user": {"login": "alice"}, "body": "I authored this PR", "id": 2},
|
||||
{"user": {"login": "random-user"}, "body": "Looks okay to me", "id": 3},
|
||||
])
|
||||
if sc == "T16_comments_generic_approval":
|
||||
return self._json(200, [
|
||||
{"user": {"login": "core-qa-agent"}, "body": "APPROVED — all acceptance criteria met", "id": 1},
|
||||
{"user": {"login": "alice"}, "body": "-authored", "id": 2},
|
||||
])
|
||||
if sc == "T17_comments_no_approval":
|
||||
return self._json(200, [
|
||||
{"user": {"login": "alice"}, "body": "I authored this PR", "id": 1},
|
||||
{"user": {"login": "random-user"}, "body": "Looks okay to me", "id": 2},
|
||||
])
|
||||
if sc == "T18_review_wrong_team_comment_right_team":
|
||||
return self._json(200, [
|
||||
{"user": {"login": "core-qa-agent"}, "body": "[core-qa-agent] APPROVED after focused review", "id": 1},
|
||||
])
|
||||
# Default scenarios (T1–T9, T14): no comments
|
||||
return self._json(200, [])
|
||||
|
||||
# GET /teams/{team_id}/members/{username}
|
||||
m = re.match(r"^/api/v1/teams/(\d+)/members/([^/]+)$", path)
|
||||
if m:
|
||||
login = m.group(2)
|
||||
team_id, login = m.group(1), m.group(2)
|
||||
if sc == "T8_team_not_member":
|
||||
return self._empty(404)
|
||||
if sc == "T9_team_403":
|
||||
return self._empty(403)
|
||||
if sc == "T18_review_wrong_team_comment_right_team" and login == "core-devops":
|
||||
return self._empty(404)
|
||||
if sc == "T19_ai_sop_ack_approved" and login == "ai-reviewer":
|
||||
# ai-sop-ack member is NOT in qa (20) or security (21).
|
||||
return self._empty(404)
|
||||
# T7_team_member: member
|
||||
return self._empty(204)
|
||||
|
||||
# GET /repos/{owner}/{name}/statuses/{sha} — for N/A declaration check
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/statuses/([a-f0-9]+)$", path)
|
||||
if m:
|
||||
# All comment-based scenarios have no N/A declarations
|
||||
return self._json(200, [])
|
||||
|
||||
return self._json(404, {"path": path, "msg": "fixture: no route"})
|
||||
|
||||
def do_POST(self):
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "ci-required-drift.py"
|
||||
spec = importlib.util.spec_from_file_location("ci_required_drift", SCRIPT)
|
||||
drift = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = drift
|
||||
spec.loader.exec_module(drift)
|
||||
|
||||
# Module-level constants are loaded from env at import time; set them
|
||||
# explicitly so unit tests can import without the full env contract.
|
||||
drift.SENTINEL_JOB = "all-required"
|
||||
drift.CI_WORKFLOW_PATH = ".gitea/workflows/ci.yml"
|
||||
drift.AUDIT_WORKFLOW_PATH = ".gitea/workflows/audit-force-merge.yml"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_ci_doc(jobs: dict) -> dict:
|
||||
return {"jobs": jobs}
|
||||
|
||||
|
||||
def _make_audit_doc(required_checks: list[str]) -> dict:
|
||||
return {
|
||||
"jobs": {
|
||||
"audit": {
|
||||
"steps": [
|
||||
{"env": {"REQUIRED_CHECKS": "\n".join(required_checks)}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sentinel_needs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_sentinel_needs_returns_empty_when_absent():
|
||||
doc = _make_ci_doc({"all-required": {"runs-on": "ubuntu-latest"}})
|
||||
assert drift.sentinel_needs(doc) == set()
|
||||
|
||||
|
||||
def test_sentinel_needs_parses_list():
|
||||
doc = _make_ci_doc(
|
||||
{"all-required": {"needs": ["platform-build", "canvas-build"]}}
|
||||
)
|
||||
assert drift.sentinel_needs(doc) == {"platform-build", "canvas-build"}
|
||||
|
||||
|
||||
def test_sentinel_needs_parses_string():
|
||||
doc = _make_ci_doc({"all-required": {"needs": "platform-build"}})
|
||||
assert drift.sentinel_needs(doc) == {"platform-build"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci_job_names / ci_jobs_all
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_ci_job_names_excludes_sentinel_and_event_gated():
|
||||
doc = _make_ci_doc(
|
||||
{
|
||||
"platform-build": {},
|
||||
"canvas-build": {"if": "github.event_name == 'pull_request'"},
|
||||
"main-push": {"if": "github.ref == 'refs/heads/main'"},
|
||||
"all-required": {},
|
||||
}
|
||||
)
|
||||
assert drift.ci_job_names(doc) == {"platform-build"}
|
||||
|
||||
|
||||
def test_ci_jobs_all_includes_event_gated():
|
||||
doc = _make_ci_doc(
|
||||
{
|
||||
"platform-build": {},
|
||||
"canvas-build": {"if": "github.event_name == 'pull_request'"},
|
||||
"all-required": {},
|
||||
}
|
||||
)
|
||||
assert drift.ci_jobs_all(doc) == {"platform-build", "canvas-build"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# detect_drift — F1 / F1b with mocked I/O
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SAMPLE_PROTECTION = {
|
||||
"status_check_contexts": [
|
||||
"CI / all-required (pull_request)",
|
||||
"Secret scan / Scan diff for credential-shaped strings (pull_request)",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_detect_drift_no_needs_sentinel_skips_f1():
|
||||
"""Post-#1766 contract: all-required has no needs: → F1 is a false positive."""
|
||||
ci = _make_ci_doc(
|
||||
{
|
||||
"platform-build": {},
|
||||
"canvas-build": {},
|
||||
"all-required": {},
|
||||
}
|
||||
)
|
||||
audit = _make_audit_doc(
|
||||
[
|
||||
"CI / all-required (pull_request)",
|
||||
"Secret scan / Scan diff for credential-shaped strings (pull_request)",
|
||||
]
|
||||
)
|
||||
|
||||
with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
|
||||
with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
|
||||
findings, debug = drift.detect_drift("main")
|
||||
|
||||
assert findings == []
|
||||
assert debug["sentinel_needs"] == []
|
||||
|
||||
|
||||
def test_detect_drift_typo_in_needs_triggers_f1b():
|
||||
"""F1b still catches typos when needs exists."""
|
||||
ci = _make_ci_doc(
|
||||
{
|
||||
"platform-build": {},
|
||||
"all-required": {"needs": ["platfom-build"]}, # typo
|
||||
}
|
||||
)
|
||||
audit = _make_audit_doc(["CI / all-required (pull_request)"])
|
||||
|
||||
with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
|
||||
with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
|
||||
findings, _ = drift.detect_drift("main")
|
||||
|
||||
assert any("F1b" in f for f in findings)
|
||||
assert any("platfom-build" in f for f in findings)
|
||||
|
||||
|
||||
def test_detect_drift_missing_job_in_needs_triggers_f1():
|
||||
"""F1 still fires when needs is non-empty and jobs are missing."""
|
||||
ci = _make_ci_doc(
|
||||
{
|
||||
"platform-build": {},
|
||||
"canvas-build": {},
|
||||
"all-required": {"needs": ["platform-build"]},
|
||||
}
|
||||
)
|
||||
audit = _make_audit_doc(["CI / all-required (pull_request)"])
|
||||
|
||||
with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
|
||||
with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
|
||||
findings, _ = drift.detect_drift("main")
|
||||
|
||||
assert any("F1 —" in f for f in findings)
|
||||
assert any("canvas-build" in f for f in findings)
|
||||
assert not any("F1b" in f for f in findings)
|
||||
|
||||
|
||||
def test_detect_drift_no_f1_when_needs_empty_even_with_jobs():
|
||||
"""Explicit regression guard: empty needs + existing jobs = no F1."""
|
||||
ci = _make_ci_doc(
|
||||
{
|
||||
"platform-build": {},
|
||||
"canvas-build": {},
|
||||
"all-required": {"needs": []},
|
||||
}
|
||||
)
|
||||
audit = _make_audit_doc(["CI / all-required (pull_request)"])
|
||||
|
||||
with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
|
||||
with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
|
||||
findings, _ = drift.detect_drift("main")
|
||||
|
||||
assert not any("F1 —" in f for f in findings)
|
||||
@@ -1,110 +0,0 @@
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_workflow(name: str) -> dict:
|
||||
with (ROOT / "workflows" / name).open() as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def _all_required(workflow: dict) -> dict:
|
||||
return workflow["jobs"]["all-required"]
|
||||
|
||||
|
||||
def test_all_required_uses_dedicated_meta_runner_lane():
|
||||
workflow = load_workflow("ci.yml")
|
||||
all_required = _all_required(workflow)
|
||||
|
||||
# Stays on the dedicated `ci-meta` lane (the sentinel does no docker
|
||||
# work, so it must NOT occupy the general docker-host pool).
|
||||
assert all_required["runs-on"] == "ci-meta"
|
||||
|
||||
|
||||
def test_all_required_is_needs_aggregator_not_a_polling_gate():
|
||||
"""fix/ci-scheduler-fanout (2026-06-01): the sentinel was converted
|
||||
from a status-polling loop (which squatted a ci-meta executor slot for
|
||||
up to 40 min per PR) into a plain `needs:` aggregator that frees the
|
||||
slot immediately. Pin the new shape so a regression to the poller is
|
||||
caught.
|
||||
"""
|
||||
workflow = load_workflow("ci.yml")
|
||||
all_required = _all_required(workflow)
|
||||
rendered = str(all_required)
|
||||
|
||||
# The job MUST aggregate via `needs:` (the slot-freeing design).
|
||||
assert "needs" in all_required, "all-required must be a needs: aggregator"
|
||||
|
||||
# It MUST NOT reintroduce the polling loop / per-SHA status fetch that
|
||||
# was the throughput sink.
|
||||
assert "detect-changes.py" not in rendered, (
|
||||
"all-required must not run the detect-changes poller path"
|
||||
)
|
||||
assert "commits/" not in rendered and "statuses" not in rendered, (
|
||||
"all-required must not poll commit statuses (the slot-squat path)"
|
||||
)
|
||||
|
||||
|
||||
def test_all_required_does_not_use_if_always():
|
||||
"""Plain `needs:` works on Gitea 1.22.6 / act_runner v0.6.1; `needs:` +
|
||||
`if: always()` is BROKEN (feedback_gitea_needs_works_only_ifalways_broken)
|
||||
and would let a non-success need pass the gate. The sentinel must use
|
||||
plain `needs:` WITHOUT a job-level `if: always()`.
|
||||
"""
|
||||
workflow = load_workflow("ci.yml")
|
||||
all_required = _all_required(workflow)
|
||||
|
||||
job_if = all_required.get("if")
|
||||
assert not (isinstance(job_if, str) and "always()" in job_if), (
|
||||
"all-required must not combine needs: with if: always()"
|
||||
)
|
||||
|
||||
|
||||
def test_all_required_needs_matches_ci_required_drift_f1_set():
|
||||
"""The sentinel `needs:` list MUST equal ci-required-drift.py's
|
||||
`ci_job_names()` set: every job MINUS the sentinel itself MINUS jobs
|
||||
whose `if:` gates on github.event_name/github.ref (event-gated jobs
|
||||
skip on PRs and a `needs:` on a skipped job would never let the
|
||||
sentinel run). If they diverge, ci-required-drift F1 fires.
|
||||
"""
|
||||
workflow = load_workflow("ci.yml")
|
||||
jobs = workflow["jobs"]
|
||||
sentinel = "all-required"
|
||||
|
||||
expected = set()
|
||||
for key, body in jobs.items():
|
||||
if key == sentinel:
|
||||
continue
|
||||
gate = body.get("if") if isinstance(body, dict) else None
|
||||
if isinstance(gate, str) and (
|
||||
"github.event_name" in gate or "github.ref" in gate
|
||||
):
|
||||
# event-gated → legitimately skips on some triggers; excluded
|
||||
# from both `needs:` and the F1 set.
|
||||
continue
|
||||
expected.add(key)
|
||||
|
||||
needs = jobs[sentinel].get("needs", [])
|
||||
if isinstance(needs, str):
|
||||
needs = [needs]
|
||||
actual = set(needs)
|
||||
|
||||
assert actual == expected, (
|
||||
f"all-required needs: {sorted(actual)} != ci_job_names() "
|
||||
f"{sorted(expected)} — ci-required-drift F1 would fire"
|
||||
)
|
||||
|
||||
|
||||
def test_all_required_needs_reference_real_jobs():
|
||||
"""F1b guard: every entry in `needs:` must name an existing job."""
|
||||
workflow = load_workflow("ci.yml")
|
||||
jobs = workflow["jobs"]
|
||||
needs = jobs["all-required"].get("needs", [])
|
||||
if isinstance(needs, str):
|
||||
needs = [needs]
|
||||
job_keys = set(jobs)
|
||||
for dep in needs:
|
||||
assert dep in job_keys, f"all-required needs unknown job {dep!r}"
|
||||
@@ -1,168 +0,0 @@
|
||||
"""Regression test #765 — gate auto-fire on real qa/security APPROVED review.
|
||||
|
||||
Validates the structural configuration of qa-review.yml and security-review.yml
|
||||
so that a real team-member APPROVED review fires the workflow and POSTs the
|
||||
exact branch-protection-required context name. This is the test #2020's
|
||||
stale-context failure would have caught.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_workflow(name: str) -> dict:
|
||||
with (ROOT / "workflows" / name).open() as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def _job_guard_string(workflow: dict) -> str:
|
||||
"""Return the raw job-level `if:` string for the single job."""
|
||||
jobs = workflow["jobs"]
|
||||
# Both qa-review and security-review have exactly one job named "approved".
|
||||
job = jobs["approved"]
|
||||
return str(job.get("if", ""))
|
||||
|
||||
|
||||
def _post_step(workflow: dict) -> dict:
|
||||
"""Return the explicit POST /statuses step from the job steps list."""
|
||||
jobs = workflow["jobs"]
|
||||
steps = jobs["approved"]["steps"]
|
||||
for step in steps:
|
||||
name = step.get("name", "")
|
||||
if "Post required status context" in name:
|
||||
return step
|
||||
raise AssertionError("No explicit POST status step found")
|
||||
|
||||
|
||||
class TestQaReviewDirectTrigger:
|
||||
def test_trigger_is_pull_request_review_submitted(self):
|
||||
wf = load_workflow("qa-review.yml")
|
||||
# PyYAML parses bare 'on' as boolean True.
|
||||
on = wf[True]
|
||||
assert "pull_request_review" in on, (
|
||||
"qa-review must trigger on pull_request_review"
|
||||
)
|
||||
types = on["pull_request_review"].get("types", [])
|
||||
assert "submitted" in types, (
|
||||
"pull_request_review must include 'submitted' type"
|
||||
)
|
||||
|
||||
def test_job_guard_requires_approved_state(self):
|
||||
wf = load_workflow("qa-review.yml")
|
||||
guard = _job_guard_string(wf)
|
||||
assert "github.event.review.state == 'APPROVED'" in guard, (
|
||||
"job guard must check review.state for 'APPROVED'"
|
||||
)
|
||||
assert "github.event.review.state == 'approved'" in guard, (
|
||||
"job guard must check review.state for 'approved' (case fallback per #2135)"
|
||||
)
|
||||
|
||||
def test_post_step_uses_status_post_token(self):
|
||||
wf = load_workflow("qa-review.yml")
|
||||
post = _post_step(wf)
|
||||
env = post.get("env", {})
|
||||
assert env.get("GITEA_TOKEN") == "${{ secrets.STATUS_POST_TOKEN }}", (
|
||||
"POST step must use STATUS_POST_TOKEN for write-scoped status POST"
|
||||
)
|
||||
|
||||
def test_post_step_context_name_exact(self):
|
||||
"""The context POSTed must byte-match the branch-protection requirement."""
|
||||
wf = load_workflow("qa-review.yml")
|
||||
post = _post_step(wf)
|
||||
run = post.get("run", "")
|
||||
assert '"qa-review / approved (pull_request_target)"' in run, (
|
||||
"POST step must emit exact BP-required context name"
|
||||
)
|
||||
|
||||
|
||||
class TestSecurityReviewDirectTrigger:
|
||||
def test_trigger_is_pull_request_review_submitted(self):
|
||||
wf = load_workflow("security-review.yml")
|
||||
# PyYAML parses bare 'on' as boolean True.
|
||||
on = wf[True]
|
||||
assert "pull_request_review" in on, (
|
||||
"security-review must trigger on pull_request_review"
|
||||
)
|
||||
types = on["pull_request_review"].get("types", [])
|
||||
assert "submitted" in types, (
|
||||
"pull_request_review must include 'submitted' type"
|
||||
)
|
||||
|
||||
def test_job_guard_requires_approved_state(self):
|
||||
wf = load_workflow("security-review.yml")
|
||||
guard = _job_guard_string(wf)
|
||||
assert "github.event.review.state == 'APPROVED'" in guard, (
|
||||
"job guard must check review.state for 'APPROVED'"
|
||||
)
|
||||
assert "github.event.review.state == 'approved'" in guard, (
|
||||
"job guard must check review.state for 'approved' (case fallback per #2135)"
|
||||
)
|
||||
|
||||
def test_post_step_uses_status_post_token(self):
|
||||
wf = load_workflow("security-review.yml")
|
||||
post = _post_step(wf)
|
||||
env = post.get("env", {})
|
||||
assert env.get("GITEA_TOKEN") == "${{ secrets.STATUS_POST_TOKEN }}", (
|
||||
"POST step must use STATUS_POST_TOKEN for write-scoped status POST"
|
||||
)
|
||||
|
||||
def test_post_step_context_name_exact(self):
|
||||
"""The context POSTed must byte-match the branch-protection requirement."""
|
||||
wf = load_workflow("security-review.yml")
|
||||
post = _post_step(wf)
|
||||
run = post.get("run", "")
|
||||
assert '"security-review / approved (pull_request_target)"' in run, (
|
||||
"POST step must emit exact BP-required context name"
|
||||
)
|
||||
|
||||
|
||||
class TestRefireScriptContextName:
|
||||
"""review-refire-status.sh must emit the BP-required (pull_request_target) context."""
|
||||
|
||||
def test_refire_script_context_is_pull_request_target(self):
|
||||
script = ROOT / "scripts" / "review-refire-status.sh"
|
||||
content = script.read_text()
|
||||
assert 'CONTEXT="${TEAM}-review / approved (pull_request_target)"' in content, (
|
||||
"refire script CONTEXT must be the exact BP-required (pull_request_target) variant"
|
||||
)
|
||||
assert 'approved (pull_request)"' not in content, (
|
||||
"refire script must NOT post bare (pull_request) context"
|
||||
)
|
||||
|
||||
|
||||
class TestRefireTokenSeparation:
|
||||
"""The /qa-recheck + /security-recheck backstop must also use STATUS_POST_TOKEN."""
|
||||
|
||||
def _refire_step(self, workflow_name: str, step_name_keyword: str) -> dict:
|
||||
wf = load_workflow(workflow_name)
|
||||
jobs = wf["jobs"]
|
||||
steps = jobs["review-refire"]["steps"]
|
||||
for step in steps:
|
||||
name = step.get("name", "")
|
||||
if step_name_keyword in name:
|
||||
return step
|
||||
raise AssertionError(f"No refire step matching {step_name_keyword!r}")
|
||||
|
||||
def test_qa_refire_uses_status_post_token(self):
|
||||
step = self._refire_step("sop-checklist.yml", "Refire qa-review")
|
||||
env = step.get("env", {})
|
||||
assert env.get("STATUS_POST_TOKEN") == "${{ secrets.STATUS_POST_TOKEN }}", (
|
||||
"qa refire must receive STATUS_POST_TOKEN env var"
|
||||
)
|
||||
# Evaluator stays on read token
|
||||
assert "SOP_TIER_CHECK_TOKEN" in env.get("GITEA_TOKEN", "") or "GITHUB_TOKEN" in env.get("GITEA_TOKEN", ""), (
|
||||
"qa refire evaluator must stay on read-scoped token"
|
||||
)
|
||||
|
||||
def test_security_refire_uses_status_post_token(self):
|
||||
step = self._refire_step("sop-checklist.yml", "Refire security-review")
|
||||
env = step.get("env", {})
|
||||
assert env.get("STATUS_POST_TOKEN") == "${{ secrets.STATUS_POST_TOKEN }}", (
|
||||
"security refire must receive STATUS_POST_TOKEN env var"
|
||||
)
|
||||
assert "SOP_TIER_CHECK_TOKEN" in env.get("GITEA_TOKEN", "") or "GITHUB_TOKEN" in env.get("GITEA_TOKEN", ""), (
|
||||
"security refire evaluator must stay on read-scoped token"
|
||||
)
|
||||
@@ -1,129 +0,0 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "gitea-merge-queue.py"
|
||||
spec = importlib.util.spec_from_file_location("gitea_merge_queue", SCRIPT)
|
||||
mq = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = mq
|
||||
spec.loader.exec_module(mq)
|
||||
|
||||
|
||||
def test_latest_statuses_dedupes_by_context_newest_first():
|
||||
statuses = [
|
||||
{"context": "CI / all-required (pull_request)", "status": "failure"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "state": "success"},
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
]
|
||||
|
||||
latest = mq.latest_statuses_by_context(statuses)
|
||||
|
||||
assert latest["CI / all-required (pull_request)"]["status"] == "failure"
|
||||
assert latest["sop-checklist / all-items-acked (pull_request)"]["state"] == "success"
|
||||
|
||||
|
||||
def test_required_contexts_green_rejects_missing_and_pending():
|
||||
latest = mq.latest_statuses_by_context([
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "pending"},
|
||||
])
|
||||
|
||||
ok, missing_or_bad = mq.required_contexts_green(
|
||||
latest,
|
||||
[
|
||||
"CI / all-required (pull_request)",
|
||||
"sop-checklist / all-items-acked (pull_request)",
|
||||
"qa-review / approved (pull_request)",
|
||||
],
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert missing_or_bad == [
|
||||
"sop-checklist / all-items-acked (pull_request)=pending",
|
||||
"qa-review / approved (pull_request)=missing",
|
||||
]
|
||||
|
||||
|
||||
def test_choose_next_pr_sorts_by_queue_label_timestamp_then_number():
|
||||
issues = [
|
||||
{
|
||||
"number": 12,
|
||||
"pull_request": {},
|
||||
"labels": [{"name": "merge-queue"}],
|
||||
"created_at": "2026-05-13T05:00:00Z",
|
||||
"updated_at": "2026-05-13T06:00:00Z",
|
||||
},
|
||||
{
|
||||
"number": 9,
|
||||
"pull_request": {},
|
||||
"labels": [{"name": "merge-queue"}],
|
||||
"created_at": "2026-05-13T04:00:00Z",
|
||||
"updated_at": "2026-05-13T07:00:00Z",
|
||||
},
|
||||
{
|
||||
"number": 7,
|
||||
"labels": [{"name": "merge-queue"}],
|
||||
"created_at": "2026-05-13T03:00:00Z",
|
||||
},
|
||||
]
|
||||
|
||||
selected = mq.choose_next_queued_issue(issues, queue_label="merge-queue")
|
||||
|
||||
assert selected["number"] == 9
|
||||
|
||||
|
||||
def test_pr_needs_update_when_base_sha_absent_from_commits():
|
||||
commits = [
|
||||
{"sha": "head"},
|
||||
{"sha": "parent"},
|
||||
]
|
||||
|
||||
assert mq.pr_contains_base_sha(commits, "mainsha") is False
|
||||
assert mq.pr_contains_base_sha(commits, "parent") is True
|
||||
|
||||
|
||||
def test_merge_decision_requires_main_green_pr_green_and_current_base():
|
||||
required = ["CI / all-required (pull_request)"]
|
||||
main_status = {
|
||||
"state": "success",
|
||||
"statuses": [{"context": "CI / all-required (push)", "status": "success"}],
|
||||
}
|
||||
pr_status = {
|
||||
"state": "success",
|
||||
"statuses": [{"context": "CI / all-required (pull_request)", "status": "success"}],
|
||||
}
|
||||
|
||||
decision = mq.evaluate_merge_readiness(
|
||||
main_status=main_status,
|
||||
pr_status=pr_status,
|
||||
required_contexts=required,
|
||||
pr_has_current_base=True,
|
||||
)
|
||||
|
||||
assert decision.ready is True
|
||||
assert decision.action == "merge"
|
||||
|
||||
|
||||
def test_merge_decision_updates_stale_pr_before_merge():
|
||||
decision = mq.evaluate_merge_readiness(
|
||||
main_status={
|
||||
"state": "success",
|
||||
"statuses": [{"context": "CI / all-required (push)", "status": "success"}],
|
||||
},
|
||||
pr_status={"state": "success", "statuses": [{"context": "CI / all-required (pull_request)", "status": "success"}]},
|
||||
required_contexts=["CI / all-required (pull_request)"],
|
||||
pr_has_current_base=False,
|
||||
)
|
||||
|
||||
assert decision.ready is False
|
||||
assert decision.action == "update"
|
||||
|
||||
|
||||
def test_MergePermissionError_inherits_from_ApiError():
|
||||
assert issubclass(mq.MergePermissionError, mq.ApiError)
|
||||
|
||||
|
||||
def test_MergePermissionError_message_preserved():
|
||||
exc = mq.MergePermissionError("POST /merge -> HTTP 405: User not allowed")
|
||||
assert "405" in str(exc)
|
||||
assert "User not allowed" in str(exc)
|
||||
@@ -15,6 +15,7 @@ Mirrors the pattern in scripts/ops/test_check_migration_collisions.py
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
@@ -1,283 +0,0 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "main-red-watchdog.py"
|
||||
spec = importlib.util.spec_from_file_location("main_red_watchdog", SCRIPT)
|
||||
wd = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = wd
|
||||
spec.loader.exec_module(wd)
|
||||
|
||||
# Module-level constants are loaded from env at import time; set them
|
||||
# explicitly so unit tests can import without the full env contract.
|
||||
wd.GITEA_TOKEN = "fake-token"
|
||||
wd.GITEA_HOST = "git.example.com"
|
||||
wd.REPO = "molecule-ai/molecule-core"
|
||||
wd.OWNER = "molecule-ai"
|
||||
wd.NAME = "molecule-core"
|
||||
wd.WATCH_BRANCH = "main"
|
||||
wd.RED_LABEL = "tier:high"
|
||||
wd.API = "https://git.example.com/api/v1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _is_scheduled_context
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_is_scheduled_context_matches_staging_saas_smoke():
|
||||
assert wd._is_scheduled_context("Staging SaaS smoke") is True
|
||||
|
||||
|
||||
def test_is_scheduled_context_matches_case_insensitive():
|
||||
assert wd._is_scheduled_context("continuous synthetic e2e") is True
|
||||
|
||||
|
||||
def test_is_scheduled_context_no_match_for_required_ci():
|
||||
assert wd._is_scheduled_context("CI / all-required") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _entry_state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_entry_state_prefers_status_over_state():
|
||||
"""Gitea 1.22.6 per-entry key is `status`; `state` is fallback."""
|
||||
assert wd._entry_state({"status": "failure", "state": "success"}) == "failure"
|
||||
|
||||
|
||||
def test_entry_state_falls_back_to_state():
|
||||
assert wd._entry_state({"state": "pending"}) == "pending"
|
||||
|
||||
|
||||
def test_entry_state_empty_when_neither_key_present():
|
||||
assert wd._entry_state({"context": "foo"}) == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_red
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_is_red_combined_failure_no_statuses():
|
||||
"""Combined failure with empty statuses[] still trips red."""
|
||||
red, failed = wd.is_red({"state": "failure", "statuses": []})
|
||||
assert red is True
|
||||
assert failed == []
|
||||
|
||||
|
||||
def test_is_red_cancel_cascade_filtered():
|
||||
"""status=3 (cancelled) mapped to failure string must be filtered."""
|
||||
status = {
|
||||
"state": "failure",
|
||||
"statuses": [
|
||||
{"context": "CI / build", "status": "failure", "description": "Has been cancelled"},
|
||||
],
|
||||
}
|
||||
red, failed = wd.is_red(status)
|
||||
assert red is False
|
||||
assert failed == []
|
||||
|
||||
|
||||
def test_is_red_real_failure_not_filtered():
|
||||
"""Real failures with different descriptions are kept."""
|
||||
status = {
|
||||
"state": "failure",
|
||||
"statuses": [
|
||||
{"context": "CI / build", "status": "failure", "description": "Failing after 12s"},
|
||||
],
|
||||
}
|
||||
red, failed = wd.is_red(status)
|
||||
assert red is True
|
||||
assert len(failed) == 1
|
||||
assert failed[0]["context"] == "CI / build"
|
||||
|
||||
|
||||
def test_is_red_uses_entry_state_not_top_level_state():
|
||||
"""Regression: per-entry key is `status`, not `state`."""
|
||||
status = {
|
||||
"state": "failure",
|
||||
"statuses": [
|
||||
# Only `status` present; pre-rev4 code read `state` and got None
|
||||
{"context": "CI / test", "status": "failure"},
|
||||
],
|
||||
}
|
||||
red, failed = wd.is_red(status)
|
||||
assert red is True
|
||||
assert len(failed) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# list_open_red_issues — pagination (mc#1789)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_list_open_red_issues_exhausts_pagination():
|
||||
"""Backlog can exceed 50 issues; all pages must be fetched."""
|
||||
calls = []
|
||||
|
||||
def fake_api(method, path, **kwargs):
|
||||
calls.append((method, path, kwargs))
|
||||
query = (kwargs.get("query") or {})
|
||||
page = int(query.get("page", "1"))
|
||||
limit = int(query.get("limit", "50"))
|
||||
# Page 1 returns full limit; page 2 returns partial → break
|
||||
if page == 1:
|
||||
return 200, [
|
||||
{"title": f"[main-red] molecule-ai/molecule-core: sha{i:04d}"}
|
||||
for i in range(limit)
|
||||
]
|
||||
if page == 2:
|
||||
return 200, [
|
||||
{"title": "[main-red] molecule-ai/molecule-core: extra1"},
|
||||
{"title": "[main-red] molecule-ai/molecule-core: extra2"},
|
||||
{"title": " unrelated issue "}, # filtered out
|
||||
]
|
||||
return 200, []
|
||||
|
||||
with patch.object(wd, "api", side_effect=fake_api):
|
||||
issues = wd.list_open_red_issues()
|
||||
|
||||
assert len(issues) == 52 # 50 + 2 matched
|
||||
titles = {i["title"] for i in issues}
|
||||
assert "[main-red] molecule-ai/molecule-core: extra1" in titles
|
||||
assert "[main-red] molecule-ai/molecule-core: extra2" in titles
|
||||
|
||||
|
||||
def test_list_open_red_issues_single_page():
|
||||
"""When results < limit, loop breaks after first page."""
|
||||
def fake_api(method, path, **kwargs):
|
||||
return 200, [
|
||||
{"title": "[main-red] molecule-ai/molecule-core: abc123"},
|
||||
]
|
||||
|
||||
with patch.object(wd, "api", side_effect=fake_api):
|
||||
issues = wd.list_open_red_issues()
|
||||
|
||||
assert len(issues) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_once — close logic (mc#1789)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_run_once_green_closes_stale_issues(monkeypatch):
|
||||
"""Combined success → close stale issues."""
|
||||
monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
|
||||
monkeypatch.setattr(wd, "get_combined_status", lambda s: {"state": "success", "statuses": []})
|
||||
monkeypatch.setattr(wd, "is_red", lambda s: (False, []))
|
||||
|
||||
closed = []
|
||||
|
||||
def capture_close(current_sha, *, dry_run=False, close_same_sha=False):
|
||||
closed.append(current_sha)
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", capture_close)
|
||||
monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
|
||||
|
||||
assert wd.run_once(dry_run=True) == 0
|
||||
assert closed == ["abc123"]
|
||||
|
||||
|
||||
def test_run_once_pending_scheduled_only_closes_stale_issues(monkeypatch):
|
||||
"""Combined pending, but only scheduled contexts pending → close stale."""
|
||||
monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
|
||||
monkeypatch.setattr(
|
||||
wd, "get_combined_status",
|
||||
lambda s: {
|
||||
"state": "pending",
|
||||
"statuses": [
|
||||
{"context": "CI / all-required", "status": "success"},
|
||||
{"context": "Staging SaaS smoke", "status": "pending"},
|
||||
],
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(wd, "is_red", lambda s: (False, []))
|
||||
|
||||
closed = []
|
||||
|
||||
def capture_close(current_sha, *, dry_run=False, close_same_sha=False):
|
||||
closed.append(current_sha)
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", capture_close)
|
||||
monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
|
||||
|
||||
assert wd.run_once(dry_run=True) == 0
|
||||
assert closed == ["abc123"]
|
||||
|
||||
|
||||
def test_run_once_pending_required_does_not_close(monkeypatch):
|
||||
"""Combined pending with a real required context still pending → no close."""
|
||||
monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
|
||||
monkeypatch.setattr(
|
||||
wd, "get_combined_status",
|
||||
lambda s: {
|
||||
"state": "pending",
|
||||
"statuses": [
|
||||
{"context": "CI / all-required", "status": "pending"},
|
||||
{"context": "Staging SaaS smoke", "status": "success"},
|
||||
],
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(wd, "is_red", lambda s: (False, []))
|
||||
|
||||
closed = []
|
||||
|
||||
def capture_close(current_sha, *, dry_run=False, close_same_sha=False):
|
||||
closed.append(current_sha)
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", capture_close)
|
||||
monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
|
||||
|
||||
assert wd.run_once(dry_run=True) == 0
|
||||
assert closed == []
|
||||
|
||||
|
||||
def test_run_once_failure_does_not_close(monkeypatch):
|
||||
"""Real failure in non-scheduled context → no close."""
|
||||
monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
|
||||
monkeypatch.setattr(
|
||||
wd, "get_combined_status",
|
||||
lambda s: {
|
||||
"state": "failure",
|
||||
"statuses": [
|
||||
{"context": "CI / all-required", "status": "failure"},
|
||||
],
|
||||
}
|
||||
)
|
||||
# is_red will return True, so we enter the red path, not the green close path
|
||||
monkeypatch.setattr(wd, "is_red", lambda s: (True, s.get("statuses", [])))
|
||||
monkeypatch.setattr(wd, "time", MagicMock(sleep=lambda x: None))
|
||||
monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
|
||||
|
||||
filed = []
|
||||
|
||||
def capture_file(sha, failed, debug, *, dry_run=False):
|
||||
filed.append(sha)
|
||||
|
||||
monkeypatch.setattr(wd, "file_or_update_red", capture_file)
|
||||
monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", lambda *a, **k: 0)
|
||||
monkeypatch.setattr(wd, "close_stale_red_issues", lambda *a, **k: 0)
|
||||
|
||||
assert wd.run_once(dry_run=True) == 0
|
||||
assert filed == ["abc123"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# title_for / find_open_issue_for_sha
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_title_for_uses_short_sha():
|
||||
assert wd.title_for("abcdef123456") == "[main-red] molecule-ai/molecule-core: abcdef1234"
|
||||
|
||||
|
||||
def test_find_open_issue_for_sha_matches_exact_title(monkeypatch):
|
||||
fake_issue = {"title": "[main-red] molecule-ai/molecule-core: abc1234567", "number": 42}
|
||||
monkeypatch.setattr(wd, "list_open_red_issues", lambda: [fake_issue])
|
||||
assert wd.find_open_issue_for_sha("abc1234567") == fake_issue
|
||||
|
||||
|
||||
def test_find_open_issue_for_sha_returns_none_when_no_match(monkeypatch):
|
||||
monkeypatch.setattr(wd, "list_open_red_issues", lambda: [])
|
||||
assert wd.find_open_issue_for_sha("abc123") is None
|
||||
@@ -1,488 +0,0 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "prod-auto-deploy.py"
|
||||
spec = importlib.util.spec_from_file_location("prod_auto_deploy", SCRIPT)
|
||||
prod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = prod
|
||||
spec.loader.exec_module(prod)
|
||||
|
||||
|
||||
def test_truthy_flag_accepts_operator_disable_values():
|
||||
for value in ("1", "true", "TRUE", "yes", "on", "disabled", "disable"):
|
||||
assert prod.truthy_flag(value) is True
|
||||
|
||||
for value in ("", "0", "false", "no", "off", None):
|
||||
assert prod.truthy_flag(value) is False
|
||||
|
||||
|
||||
def test_build_plan_defaults_to_staging_sha_target_and_prod_cp():
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"PROD_AUTO_DEPLOY_DISABLED": "",
|
||||
}
|
||||
)
|
||||
|
||||
assert plan["enabled"] is True
|
||||
assert plan["sha"] == "abcdef1234567890"
|
||||
assert plan["target_tag"] == "staging-abcdef1"
|
||||
assert plan["cp_url"] == "https://api.moleculesai.app"
|
||||
assert plan["body"] == {
|
||||
"target_tag": "staging-abcdef1",
|
||||
"canary_slug": "hongming",
|
||||
"soak_seconds": 60,
|
||||
"batch_size": 3,
|
||||
"dry_run": False,
|
||||
# cp#228 / task #308: fleet-wide intent must carry confirm:true.
|
||||
"confirm": True,
|
||||
}
|
||||
|
||||
|
||||
def test_build_plan_always_sets_confirm_true_for_fleet_intent():
|
||||
"""Regression guard: every plan body MUST carry confirm:true.
|
||||
|
||||
CP /cp/admin/tenants/redeploy-fleet (cp#228) returns 400 on empty
|
||||
body / {confirm:false} / {only_slugs:[]} to prevent accidental
|
||||
fleet-wide mutation. This caller is fleet-wide intent (canary +
|
||||
fan-out, no slug scoping), so the plan MUST carry confirm:true.
|
||||
Pairs with cp#228's TestRedeployFleet_EmptyBodyReturns400 +
|
||||
TestRedeployFleet_ConfirmTrueProceeds.
|
||||
"""
|
||||
plan = prod.build_plan({"GITHUB_SHA": "abcdef1234567890"})
|
||||
assert plan["body"]["confirm"] is True
|
||||
|
||||
# Operator-overridable knobs do NOT drop the ack.
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"PROD_AUTO_DEPLOY_SOAK_SECONDS": "0",
|
||||
"PROD_AUTO_DEPLOY_BATCH_SIZE": "10",
|
||||
"PROD_AUTO_DEPLOY_DRY_RUN": "true",
|
||||
"PROD_AUTO_DEPLOY_CANARY_SLUG": "",
|
||||
}
|
||||
)
|
||||
assert plan["body"]["confirm"] is True
|
||||
|
||||
|
||||
def test_build_plan_rejects_non_prod_cp_without_explicit_override():
|
||||
try:
|
||||
prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"CP_URL": "https://staging-api.moleculesai.app",
|
||||
}
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "PROD_ALLOW_NON_PROD_CP_URL=true" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected non-prod CP URL rejection")
|
||||
|
||||
|
||||
def test_build_plan_allows_non_prod_cp_only_with_override():
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"CP_URL": "https://staging-api.moleculesai.app",
|
||||
"PROD_ALLOW_NON_PROD_CP_URL": "true",
|
||||
}
|
||||
)
|
||||
|
||||
assert plan["cp_url"] == "https://staging-api.moleculesai.app"
|
||||
|
||||
|
||||
def test_build_plan_disable_flag_short_circuits_before_credentials():
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"PROD_AUTO_DEPLOY_DISABLED": "true",
|
||||
}
|
||||
)
|
||||
|
||||
assert plan["enabled"] is False
|
||||
assert plan["disabled_reason"] == "PROD_AUTO_DEPLOY_DISABLED=true"
|
||||
|
||||
|
||||
def test_latest_status_for_context_uses_first_matching_status():
|
||||
statuses = [
|
||||
{"context": "CI / all-required (push)", "status": "pending"},
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
{"context": "CI / all-required (push)", "status": "success"},
|
||||
]
|
||||
|
||||
latest = prod.latest_status_for_context(statuses, "CI / all-required (push)")
|
||||
|
||||
assert latest == {"context": "CI / all-required (push)", "status": "pending"}
|
||||
|
||||
|
||||
def test_ci_context_state_handles_missing_and_gitea_status_key():
|
||||
assert prod.ci_context_state([], "CI / all-required (push)") == "missing"
|
||||
assert (
|
||||
prod.ci_context_state(
|
||||
[{"context": "CI / all-required (push)", "status": "success"}],
|
||||
"CI / all-required (push)",
|
||||
)
|
||||
== "success"
|
||||
)
|
||||
assert (
|
||||
prod.ci_context_state(
|
||||
[{"context": "CI / all-required (push)", "state": "failure"}],
|
||||
"CI / all-required (push)",
|
||||
)
|
||||
== "failure"
|
||||
)
|
||||
|
||||
|
||||
def test_context_is_satisfied_accepts_only_success():
|
||||
assert prod.context_is_satisfied("success") is True
|
||||
for state in ("failure", "error", "cancelled", "canceled", "skipped", "pending", "missing"):
|
||||
assert prod.context_is_satisfied(state) is False
|
||||
|
||||
|
||||
def test_context_is_terminal_failure_rejects_cancelled_and_skipped():
|
||||
for state in ("failure", "error", "cancelled", "canceled", "skipped"):
|
||||
assert prod.context_is_terminal_failure(state) is True
|
||||
for state in ("pending", "missing", "success"):
|
||||
assert prod.context_is_terminal_failure(state) is False
|
||||
|
||||
|
||||
def test_default_required_contexts_delegate_path_gating_to_all_required():
|
||||
assert prod.required_contexts({}) == [
|
||||
"CI / all-required (push)",
|
||||
"Secret scan / Scan diff for credential-shaped strings (push)",
|
||||
]
|
||||
|
||||
|
||||
def test_slugs_from_redeploy_response_uses_controlplane_plan_rows():
|
||||
body = {
|
||||
"results": [
|
||||
{"slug": "hongming", "phase": "canary", "ssm_status": "DryRun"},
|
||||
{"slug": "tenant-a", "phase": "batch-1", "ssm_status": "DryRun"},
|
||||
{"slug": "", "phase": "batch-1", "ssm_status": "DryRun"},
|
||||
{"phase": "batch-1", "ssm_status": "DryRun"},
|
||||
]
|
||||
}
|
||||
|
||||
assert prod.slugs_from_redeploy_response(body) == ["hongming", "tenant-a"]
|
||||
|
||||
|
||||
def test_plan_rollout_slugs_asks_controlplane_for_dry_run_plan():
|
||||
calls = []
|
||||
|
||||
def fake_redeploy(_cp_url, _token, body):
|
||||
calls.append(body)
|
||||
return 200, {
|
||||
"ok": True,
|
||||
"results": [
|
||||
{"slug": "hongming", "phase": "canary", "ssm_status": "DryRun"},
|
||||
{"slug": "tenant-a", "phase": "batch-1", "ssm_status": "DryRun"},
|
||||
],
|
||||
}
|
||||
|
||||
slugs = prod.plan_rollout_slugs(
|
||||
"https://api.moleculesai.app",
|
||||
"secret",
|
||||
{
|
||||
"target_tag": "staging-abcdef1",
|
||||
"canary_slug": "hongming",
|
||||
"soak_seconds": 60,
|
||||
"batch_size": 3,
|
||||
"dry_run": False,
|
||||
"confirm": True,
|
||||
},
|
||||
redeploy=fake_redeploy,
|
||||
)
|
||||
|
||||
assert slugs == ["hongming", "tenant-a"]
|
||||
assert calls == [
|
||||
{
|
||||
"target_tag": "staging-abcdef1",
|
||||
"canary_slug": "hongming",
|
||||
"soak_seconds": 60,
|
||||
"batch_size": 3,
|
||||
"dry_run": True,
|
||||
"confirm": True,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_scoped_redeploy_body_removes_canary_and_local_soak():
|
||||
base = {
|
||||
"target_tag": "staging-abcdef1",
|
||||
"canary_slug": "hongming",
|
||||
"soak_seconds": 60,
|
||||
"batch_size": 3,
|
||||
"dry_run": False,
|
||||
"confirm": True,
|
||||
}
|
||||
|
||||
scoped = prod.scoped_redeploy_body(base, ["tenant-a", "tenant-b"])
|
||||
|
||||
assert scoped == {
|
||||
"target_tag": "staging-abcdef1",
|
||||
"soak_seconds": 0,
|
||||
"batch_size": 2,
|
||||
"dry_run": False,
|
||||
"confirm": True,
|
||||
"only_slugs": ["tenant-a", "tenant-b"],
|
||||
}
|
||||
|
||||
|
||||
def test_plan_scoped_rollout_preserves_canary_then_batches():
|
||||
calls, sleeps = [], []
|
||||
|
||||
def fake_list(_cp_url, _token, _body):
|
||||
return ["tenant-a", "hongming", "tenant-b", "tenant-c"]
|
||||
|
||||
def fake_redeploy(_cp_url, _token, body):
|
||||
calls.append(body)
|
||||
return 200, {
|
||||
"ok": True,
|
||||
"results": [{"slug": slug, "healthz_ok": True} for slug in body["only_slugs"]],
|
||||
}
|
||||
|
||||
aggregate = prod.execute_scoped_rollout(
|
||||
{
|
||||
"cp_url": "https://api.moleculesai.app",
|
||||
"body": {
|
||||
"target_tag": "staging-abcdef1",
|
||||
"canary_slug": "hongming",
|
||||
"soak_seconds": 60,
|
||||
"batch_size": 2,
|
||||
"dry_run": False,
|
||||
"confirm": True,
|
||||
},
|
||||
},
|
||||
token="secret",
|
||||
list_slugs=fake_list,
|
||||
redeploy=fake_redeploy,
|
||||
sleep=sleeps.append,
|
||||
)
|
||||
|
||||
assert [call["only_slugs"] for call in calls] == [
|
||||
["hongming"],
|
||||
["tenant-a", "tenant-b"],
|
||||
["tenant-c"],
|
||||
]
|
||||
assert sleeps == [60]
|
||||
assert aggregate["ok"] is True
|
||||
assert [result["slug"] for result in aggregate["results"]] == [
|
||||
"hongming",
|
||||
"tenant-a",
|
||||
"tenant-b",
|
||||
"tenant-c",
|
||||
]
|
||||
|
||||
|
||||
def test_scoped_rollout_halts_after_failed_canary():
|
||||
calls = []
|
||||
|
||||
def fake_redeploy(_cp_url, _token, body):
|
||||
calls.append(body)
|
||||
return 200, {"ok": False, "results": [{"slug": body["only_slugs"][0], "error": "bad"}]}
|
||||
|
||||
try:
|
||||
prod.execute_scoped_rollout(
|
||||
{
|
||||
"cp_url": "https://api.moleculesai.app",
|
||||
"body": {
|
||||
"target_tag": "staging-abcdef1",
|
||||
"canary_slug": "hongming",
|
||||
"soak_seconds": 60,
|
||||
"batch_size": 2,
|
||||
"dry_run": False,
|
||||
"confirm": True,
|
||||
},
|
||||
},
|
||||
token="secret",
|
||||
list_slugs=lambda _cp_url, _token, _body: ["hongming", "tenant-a"],
|
||||
redeploy=fake_redeploy,
|
||||
sleep=lambda _seconds: None,
|
||||
)
|
||||
except prod.RolloutFailed as exc:
|
||||
assert "redeploy scoped call failed" in str(exc)
|
||||
assert exc.response["ok"] is False
|
||||
assert exc.response["results"] == [{"slug": "hongming", "error": "bad"}]
|
||||
else:
|
||||
raise AssertionError("expected failed canary to halt rollout")
|
||||
|
||||
assert [call["only_slugs"] for call in calls] == [["hongming"]]
|
||||
|
||||
|
||||
def test_rollout_from_plan_file_writes_partial_response_on_failure(tmp_path):
|
||||
plan_path = tmp_path / "plan.json"
|
||||
response_path = tmp_path / "response.json"
|
||||
plan_path.write_text(
|
||||
"""
|
||||
{
|
||||
"enabled": true,
|
||||
"cp_url": "https://api.moleculesai.app",
|
||||
"body": {"target_tag": "staging-abcdef1", "confirm": true}
|
||||
}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
original = prod.execute_scoped_rollout
|
||||
|
||||
def fake_execute(_plan, _token):
|
||||
raise prod.RolloutFailed(
|
||||
"redeploy scoped call failed for hongming: HTTP 500, ok=false",
|
||||
{
|
||||
"ok": False,
|
||||
"error": "redeploy scoped call failed for hongming: HTTP 500, ok=false",
|
||||
"results": [{"slug": "hongming", "error": "bad"}],
|
||||
},
|
||||
)
|
||||
|
||||
prod.execute_scoped_rollout = fake_execute
|
||||
try:
|
||||
try:
|
||||
prod.rollout_from_plan_file(
|
||||
str(plan_path),
|
||||
str(response_path),
|
||||
{"CP_ADMIN_API_TOKEN": "secret"},
|
||||
)
|
||||
except prod.RolloutFailed:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("expected rollout failure")
|
||||
finally:
|
||||
prod.execute_scoped_rollout = original
|
||||
|
||||
assert response_path.read_text(encoding="utf-8").strip()
|
||||
assert '"ok": false' in response_path.read_text(encoding="utf-8")
|
||||
assert '"slug": "hongming"' in response_path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# No-silent-skip coverage gate (internal#724)
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_rollout_stragglers_flags_tenant_not_on_target():
|
||||
# b SSM-succeeded but its container is on the old tag → straggler.
|
||||
stragglers = prod.rollout_stragglers(
|
||||
["a", "b", "c"],
|
||||
[
|
||||
{"slug": "a", "verified_on_target": True},
|
||||
{"slug": "b", "verified_on_target": False, "running_image": "platform-tenant:staging-old"},
|
||||
{"slug": "c", "verified_on_target": True},
|
||||
],
|
||||
)
|
||||
assert stragglers == ["b"]
|
||||
|
||||
|
||||
def test_rollout_stragglers_flags_enumerated_tenant_with_no_result():
|
||||
# agents-team class: enumerated but no batch ever produced a row for it.
|
||||
stragglers = prod.rollout_stragglers(
|
||||
["a", "agents-team"],
|
||||
[{"slug": "a", "verified_on_target": True}],
|
||||
)
|
||||
assert stragglers == ["agents-team"]
|
||||
|
||||
|
||||
def test_rollout_stragglers_missing_key_is_backward_compatible():
|
||||
# Older CP without verified_on_target → treat as verified (no spurious fail).
|
||||
stragglers = prod.rollout_stragglers(
|
||||
["a", "b"],
|
||||
[{"slug": "a", "healthz_ok": True}, {"slug": "b", "healthz_ok": True}],
|
||||
)
|
||||
assert stragglers == []
|
||||
|
||||
|
||||
def test_rollout_stragglers_ignores_dry_run_rows():
|
||||
stragglers = prod.rollout_stragglers(
|
||||
["a"], [{"slug": "a", "ssm_status": "DryRun"}]
|
||||
)
|
||||
# dry-run row is skipped, so "a" has no verifying row → straggler.
|
||||
assert stragglers == ["a"]
|
||||
|
||||
|
||||
def test_scoped_rollout_fails_when_a_tenant_stays_on_old_tag():
|
||||
# Every per-tenant call returns ok=True, but agents-team is NOT
|
||||
# verified_on_target. The rollout must still fail loudly — this is
|
||||
# the exact "reported success, one tenant silently skipped" bug.
|
||||
def fake_redeploy(_cp_url, _token, body):
|
||||
rows = []
|
||||
for slug in body["only_slugs"]:
|
||||
rows.append({"slug": slug, "verified_on_target": slug != "agents-team"})
|
||||
return 200, {"ok": True, "results": rows}
|
||||
|
||||
try:
|
||||
prod.execute_scoped_rollout(
|
||||
{
|
||||
"cp_url": "https://api.moleculesai.app",
|
||||
"body": {
|
||||
"target_tag": "staging-new",
|
||||
"batch_size": 5,
|
||||
"dry_run": False,
|
||||
"confirm": True,
|
||||
},
|
||||
},
|
||||
token="secret",
|
||||
list_slugs=lambda _u, _t, _b: ["reno-stars", "agents-team", "hongming"],
|
||||
redeploy=fake_redeploy,
|
||||
sleep=lambda _s: None,
|
||||
)
|
||||
except prod.RolloutFailed as exc:
|
||||
assert "incomplete rollout" in str(exc)
|
||||
assert exc.response["stragglers"] == ["agents-team"]
|
||||
assert exc.response["ok"] is False
|
||||
else:
|
||||
raise AssertionError("expected an incomplete rollout to fail loudly")
|
||||
|
||||
|
||||
def test_scoped_rollout_passes_when_all_tenants_verified_on_target():
|
||||
def fake_redeploy(_cp_url, _token, body):
|
||||
return 200, {
|
||||
"ok": True,
|
||||
"results": [{"slug": s, "verified_on_target": True} for s in body["only_slugs"]],
|
||||
}
|
||||
|
||||
aggregate = prod.execute_scoped_rollout(
|
||||
{
|
||||
"cp_url": "https://api.moleculesai.app",
|
||||
"body": {
|
||||
"target_tag": "staging-new",
|
||||
"batch_size": 5,
|
||||
"dry_run": False,
|
||||
"confirm": True,
|
||||
},
|
||||
},
|
||||
token="secret",
|
||||
list_slugs=lambda _u, _t, _b: ["reno-stars", "agents-team", "hongming"],
|
||||
redeploy=fake_redeploy,
|
||||
sleep=lambda _s: None,
|
||||
)
|
||||
assert aggregate["ok"] is True
|
||||
assert "stragglers" not in aggregate
|
||||
|
||||
|
||||
def test_scoped_rollout_dry_run_does_not_assert_coverage():
|
||||
# A dry run proves nothing landed; coverage must NOT be asserted or
|
||||
# every plan would fail.
|
||||
def fake_redeploy(_cp_url, _token, body):
|
||||
return 200, {
|
||||
"ok": True,
|
||||
"results": [{"slug": s, "ssm_status": "DryRun"} for s in body["only_slugs"]],
|
||||
}
|
||||
|
||||
aggregate = prod.execute_scoped_rollout(
|
||||
{
|
||||
"cp_url": "https://api.moleculesai.app",
|
||||
"body": {
|
||||
"target_tag": "staging-new",
|
||||
"batch_size": 5,
|
||||
"dry_run": True,
|
||||
"confirm": True,
|
||||
},
|
||||
},
|
||||
token="secret",
|
||||
list_slugs=lambda _u, _t, _b: ["a", "b"],
|
||||
redeploy=fake_redeploy,
|
||||
sleep=lambda _s: None,
|
||||
)
|
||||
assert aggregate["ok"] is True
|
||||
@@ -1,5 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC2034
|
||||
# Regression tests for .gitea/scripts/review-check.sh (RFC#324 Step 1).
|
||||
#
|
||||
# Covers:
|
||||
@@ -16,8 +15,6 @@
|
||||
# T11 — bash syntax check (bash -n passes)
|
||||
# T12 — jq filter: non-author APPROVED → in candidate list; dismissed → excluded
|
||||
# T13 — missing required env GITEA_TOKEN → exits 1 with error
|
||||
# T14 — non-default-base PR exits 0 without requiring review
|
||||
# T18 — wrong-team review candidate does not block right-team comment approval
|
||||
#
|
||||
# Hostile-self-review (per feedback_assert_exact_not_substring):
|
||||
# this test MUST FAIL if the script is absent. Verified by running
|
||||
@@ -76,7 +73,7 @@ assert_file_mode() {
|
||||
return
|
||||
fi
|
||||
local got_mode
|
||||
got_mode=$(stat -c '%a' "$path" 2>/dev/null || stat -f '%Lp' "$path" 2>/dev/null || echo "000")
|
||||
got_mode=$(stat -c '%a' "$path" 2>/dev/null || echo "000")
|
||||
if [ "$expected_mode" = "$got_mode" ]; then
|
||||
echo " PASS $label (mode=$got_mode)"
|
||||
PASS=$((PASS + 1))
|
||||
@@ -140,7 +137,7 @@ fi
|
||||
echo
|
||||
echo "== T13 missing GITEA_TOKEN =="
|
||||
set +e
|
||||
T13_OUT=$(PATH="/tmp:$PATH" GITEA_TOKEN='' GITEA_HOST=git.example.com REPO=x/y PR_NUMBER=1 TEAM=qa TEAM_ID=1 bash "$SCRIPT" 2>&1 || true)
|
||||
T13_OUT=$(PATH="/tmp:$PATH" GITEA_TOKEN= GITEA_HOST=git.example.com REPO=x/y PR_NUMBER=1 TEAM=qa TEAM_ID=1 bash "$SCRIPT" 2>&1 || true)
|
||||
set -e
|
||||
assert_contains "T13 exits non-zero when GITEA_TOKEN missing" "GITEA_TOKEN required" "$T13_OUT"
|
||||
|
||||
@@ -197,16 +194,13 @@ for a in "$@"; do
|
||||
done
|
||||
exec /usr/bin/curl "${new_args[@]}"
|
||||
CURL_SHIM
|
||||
# Now substitute FIXPORT with the actual port number. Use perl rather than
|
||||
# sed -i so the test runs on both GNU sed and BSD/macOS sed.
|
||||
perl -0pi -e "s/FIXPORT/${FIX_PORT}/g" "$FIXTURE_DIR/bin/curl"
|
||||
# Now substitute FIXPORT with the actual port number
|
||||
sed -i "s/FIXPORT/${FIX_PORT}/g" "$FIXTURE_DIR/bin/curl"
|
||||
chmod +x "$FIXTURE_DIR/bin/curl"
|
||||
|
||||
# Helper: run the script with fixture environment
|
||||
run_review_check() {
|
||||
local scenario="$1"
|
||||
local team="${2:-qa}"
|
||||
local team_id="${3:-20}"
|
||||
echo "$scenario" >"$FIX_STATE_DIR/scenario"
|
||||
local out
|
||||
set +e
|
||||
@@ -216,9 +210,8 @@ run_review_check() {
|
||||
GITEA_HOST="fixture.local" \
|
||||
REPO="molecule-ai/molecule-core" \
|
||||
PR_NUMBER="999" \
|
||||
DEFAULT_BRANCH="main" \
|
||||
TEAM="$team" \
|
||||
TEAM_ID="$team_id" \
|
||||
TEAM="qa" \
|
||||
TEAM_ID="20" \
|
||||
REVIEW_CHECK_DEBUG="0" \
|
||||
REVIEW_CHECK_STRICT="0" \
|
||||
bash "$SCRIPT" 2>&1
|
||||
@@ -260,14 +253,6 @@ T4_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T4 exit code 1 (no candidates)" "1" "$T4_RC"
|
||||
assert_contains "T4 awaiting non-author APPROVE" "awaiting non-author APPROVE" "$T4_OUT"
|
||||
|
||||
# T14 — non-default-base PR should not make the default branch red.
|
||||
echo
|
||||
echo "== T14 non-default base PR =="
|
||||
T14_OUT=$(run_review_check "T14_non_default_base")
|
||||
T14_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T14 exit code 0 (non-default base no-op)" "0" "$T14_RC"
|
||||
assert_contains "T14 not applicable notice" "gate not applicable" "$T14_OUT"
|
||||
|
||||
# T5 — only author reviews → exit 1
|
||||
echo
|
||||
echo "== T5 only author reviews =="
|
||||
@@ -310,12 +295,12 @@ echo
|
||||
echo "== T10 CURL_AUTH_FILE =="
|
||||
# Verify the token-file logic directly: create a temp file with the
|
||||
# same mktemp pattern, write the header with printf, chmod 600, then assert.
|
||||
T10_TOKEN="secret-fixture-token-abc123"
|
||||
T10_AUTHFILE=$(mktemp "${TMPDIR:-/tmp}/curl-auth.test.XXXXXX")
|
||||
T10_TOKEN="secret-test-token-abc123"
|
||||
T10_AUTHFILE=$(mktemp -p /tmp curl-auth.test.XXXXXX)
|
||||
chmod 600 "$T10_AUTHFILE"
|
||||
printf 'header = "Authorization: token %s"\n' "$T10_TOKEN" > "$T10_AUTHFILE"
|
||||
assert_file_mode "T10a mktemp authfile mode 600 (CURL_AUTH_FILE pattern)" "$T10_AUTHFILE" "600"
|
||||
assert_file_contains "T10b printf header format (CURL_AUTH_FILE content)" "$T10_AUTHFILE" "Authorization: token secret-fixture-token-abc123"
|
||||
assert_file_mode "T10a mktemp -p /tmp mode 600 (CURL_AUTH_FILE pattern)" "$T10_AUTHFILE" "600"
|
||||
assert_file_contains "T10b printf header format (CURL_AUTH_FILE content)" "$T10_AUTHFILE" "Authorization: token secret-test-token-abc123"
|
||||
assert_file_contains "T10c 'header =' curl-config syntax" "$T10_AUTHFILE" 'header = "Authorization: token '
|
||||
rm -f "$T10_AUTHFILE"
|
||||
|
||||
@@ -338,61 +323,6 @@ assert_contains "T12 jq: core-devops (non-author APPROVED) in candidates" "core-
|
||||
assert_eq "T12 jq: alice (author) NOT in candidates" "" "$(echo "$T12_CANDIDATES" | grep '^alice$' || true)"
|
||||
assert_eq "T12 jq: carol (dismissed) NOT in candidates" "" "$(echo "$T12_CANDIDATES" | grep '^carol$' || true)"
|
||||
|
||||
# T15 — comment-based approval via agent prefix pattern → exit 0
|
||||
echo
|
||||
echo "== T15 comment agent-prefix approval =="
|
||||
T15_OUT=$(run_review_check "T15_comments_agent_approval")
|
||||
T15_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T15 exit code 0 (agent-comment approval + team member)" "0" "$T15_RC"
|
||||
assert_contains "T15 comment fallback notice" "comment-based approval" "$T15_OUT"
|
||||
assert_contains "T15 core-qa-agent APPROVED" "APPROVED by core-qa-agent" "$T15_OUT"
|
||||
|
||||
# T16 — comment-based approval via generic APPROVED keyword → exit 0
|
||||
echo
|
||||
echo "== T16 comment generic keyword approval =="
|
||||
T16_OUT=$(run_review_check "T16_comments_generic_approval")
|
||||
T16_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T16 exit code 0 (generic-approval comment + team member)" "0" "$T16_RC"
|
||||
assert_contains "T16 comment fallback notice" "comment-based approval" "$T16_OUT"
|
||||
|
||||
# T17 — no approval keywords in comments → exit 1
|
||||
echo
|
||||
echo "== T17 comments with no approval keywords =="
|
||||
T17_OUT=$(run_review_check "T17_comments_no_approval")
|
||||
T17_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T17 exit code 1 (no candidates from comments)" "1" "$T17_RC"
|
||||
assert_contains "T17 no candidates error" "no candidates from reviews API or issue comments" "$T17_OUT"
|
||||
|
||||
# T18 — a wrong-team PR review candidate must not suppress a right-team
|
||||
# comment approval. This matches PR #1790, where QA had an APPROVED review
|
||||
# and security approved via the agent comment convention.
|
||||
echo
|
||||
echo "== T18 review candidate wrong team, comment candidate right team =="
|
||||
T18_OUT=$(run_review_check "T18_review_wrong_team_comment_right_team")
|
||||
T18_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T18 exit code 0 (comment approval still considered)" "0" "$T18_RC"
|
||||
assert_contains "T18 comment candidate notice" "comment-based approval" "$T18_OUT"
|
||||
assert_contains "T18 comment approver accepted" "APPROVED by core-qa-agent" "$T18_OUT"
|
||||
|
||||
# T19 — ai-sop-ack member APPROVED review must NOT count toward qa-review
|
||||
# or security-review (R1 hardening refinement, msg 1388c76f).
|
||||
echo
|
||||
echo "== T19 ai-sop-ack APPROVED review excluded from qa-review gate =="
|
||||
T19_OUT=$(run_review_check "T19_ai_sop_ack_approved" "qa" "20")
|
||||
T19_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T19 exit code 1 (ai-sop-ack not in qa team)" "1" "$T19_RC"
|
||||
assert_contains "T19 ai-reviewer excluded from qa" "candidates: ai-reviewer" "$T19_OUT"
|
||||
assert_contains "T19 none are in qa team" "none are in team" "$T19_OUT"
|
||||
|
||||
# T20 — same ai-sop-ack member must also be excluded from security-review gate.
|
||||
echo
|
||||
echo "== T20 ai-sop-ack APPROVED review excluded from security-review gate =="
|
||||
T20_OUT=$(run_review_check "T19_ai_sop_ack_approved" "security" "21")
|
||||
T20_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T20 exit code 1 (ai-sop-ack not in security team)" "1" "$T20_RC"
|
||||
assert_contains "T20 ai-reviewer excluded from security" "candidates: ai-reviewer" "$T20_OUT"
|
||||
assert_contains "T20 none are in security team" "none are in team" "$T20_OUT"
|
||||
|
||||
echo
|
||||
echo "------"
|
||||
echo "PASS=$PASS FAIL=$FAIL"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,524 @@
|
||||
#!/usr/bin/env python3
|
||||
# Unit tests for sop-checklist-gate.py
|
||||
#
|
||||
# Run: python3 .gitea/scripts/tests/test_sop_checklist_gate.py
|
||||
# or: pytest .gitea/scripts/tests/test_sop_checklist_gate.py
|
||||
#
|
||||
# RFC#351 Step 2 of 6 — implementation MVP. Tests cover:
|
||||
# - slug normalization (the 4 example variants in the script header)
|
||||
# - parse_directives (ack, revoke, with/without note, mid-comment, etc.)
|
||||
# - section_marker_present (empty answer rejected, filled answer ok)
|
||||
# - compute_ack_state (self-ack rejected, team probe applied, revoke
|
||||
# invalidates own prior ack, peer's ack survives unrevoked)
|
||||
# - render_status (state + description format)
|
||||
# - get_tier_mode (label-driven, default fallback)
|
||||
# - load_config (default config parses cleanly with both PyYAML and
|
||||
# the bundled minimal parser)
|
||||
#
|
||||
# All tests run WITHOUT touching the Gitea API — the team-probe
|
||||
# callable is dependency-injected.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
# Resolve sibling script regardless of where pytest is invoked from.
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT = os.path.dirname(HERE) # .gitea/scripts
|
||||
sys.path.insert(0, PARENT)
|
||||
|
||||
import importlib.util # noqa: E402
|
||||
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"sop_checklist_gate", os.path.join(PARENT, "sop-checklist-gate.py")
|
||||
)
|
||||
sop = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(sop) # type: ignore[union-attr]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CONFIG_PATH = os.path.join(PARENT, "..", "sop-checklist-config.yaml")
|
||||
|
||||
|
||||
def _items() -> list[dict]:
|
||||
cfg = sop.load_config(CONFIG_PATH)
|
||||
return cfg["items"]
|
||||
|
||||
|
||||
def _items_by_slug() -> dict[str, dict]:
|
||||
return {it["slug"]: it for it in _items()}
|
||||
|
||||
|
||||
def _numeric_aliases() -> dict[int, str]:
|
||||
return {
|
||||
int(it["numeric_alias"]): it["slug"]
|
||||
for it in _items()
|
||||
if it.get("numeric_alias")
|
||||
}
|
||||
|
||||
|
||||
def _comment(user: str, body: str) -> dict:
|
||||
return {"user": {"login": user}, "body": body}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# normalize_slug
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNormalizeSlug(unittest.TestCase):
|
||||
def test_kebab_already(self):
|
||||
self.assertEqual(sop.normalize_slug("comprehensive-testing"), "comprehensive-testing")
|
||||
|
||||
def test_underscore_to_dash(self):
|
||||
self.assertEqual(sop.normalize_slug("comprehensive_testing"), "comprehensive-testing")
|
||||
|
||||
def test_space_to_dash(self):
|
||||
self.assertEqual(sop.normalize_slug("comprehensive testing"), "comprehensive-testing")
|
||||
|
||||
def test_uppercase_to_lower(self):
|
||||
self.assertEqual(sop.normalize_slug("Comprehensive-Testing"), "comprehensive-testing")
|
||||
|
||||
def test_mixed_separators(self):
|
||||
self.assertEqual(sop.normalize_slug("Comprehensive_Testing"), "comprehensive-testing")
|
||||
self.assertEqual(sop.normalize_slug("FIVE_axis review"), "five-axis-review")
|
||||
|
||||
def test_collapse_repeated_dashes(self):
|
||||
self.assertEqual(sop.normalize_slug("comprehensive--testing"), "comprehensive-testing")
|
||||
self.assertEqual(sop.normalize_slug("comprehensive testing"), "comprehensive-testing")
|
||||
|
||||
def test_strip_trailing_punctuation(self):
|
||||
self.assertEqual(sop.normalize_slug("comprehensive-testing."), "comprehensive-testing")
|
||||
self.assertEqual(sop.normalize_slug("comprehensive-testing!"), "comprehensive-testing")
|
||||
|
||||
def test_numeric_shorthand_known(self):
|
||||
self.assertEqual(
|
||||
sop.normalize_slug("1", _numeric_aliases()),
|
||||
"comprehensive-testing",
|
||||
)
|
||||
self.assertEqual(
|
||||
sop.normalize_slug("3", _numeric_aliases()),
|
||||
"staging-smoke",
|
||||
)
|
||||
self.assertEqual(
|
||||
sop.normalize_slug("7", _numeric_aliases()),
|
||||
"memory-consulted",
|
||||
)
|
||||
|
||||
def test_numeric_shorthand_unknown_returns_empty(self):
|
||||
# "8" is out of range → empty so caller can flag as unparseable.
|
||||
self.assertEqual(sop.normalize_slug("8", _numeric_aliases()), "")
|
||||
|
||||
def test_numeric_without_alias_table_keeps_digits(self):
|
||||
# No alias table → return the digits as-is.
|
||||
self.assertEqual(sop.normalize_slug("1"), "1")
|
||||
|
||||
def test_empty_input(self):
|
||||
self.assertEqual(sop.normalize_slug(""), "")
|
||||
self.assertEqual(sop.normalize_slug(" "), "")
|
||||
self.assertEqual(sop.normalize_slug(None), "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse_directives
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseDirectives(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.aliases = _numeric_aliases()
|
||||
|
||||
def test_simple_ack(self):
|
||||
d = sop.parse_directives("/sop-ack comprehensive-testing", self.aliases)
|
||||
self.assertEqual(d, [("sop-ack", "comprehensive-testing", "")])
|
||||
|
||||
def test_simple_revoke(self):
|
||||
d = sop.parse_directives("/sop-revoke staging-smoke", self.aliases)
|
||||
self.assertEqual(d, [("sop-revoke", "staging-smoke", "")])
|
||||
|
||||
def test_ack_with_note(self):
|
||||
d = sop.parse_directives(
|
||||
"/sop-ack comprehensive-testing LGTM the test covers all edge cases",
|
||||
self.aliases,
|
||||
)
|
||||
self.assertEqual(len(d), 1)
|
||||
self.assertEqual(d[0][0], "sop-ack")
|
||||
self.assertEqual(d[0][1], "comprehensive-testing")
|
||||
self.assertIn("LGTM", d[0][2])
|
||||
|
||||
def test_numeric_shorthand(self):
|
||||
d = sop.parse_directives("/sop-ack 1", self.aliases)
|
||||
self.assertEqual(d, [("sop-ack", "comprehensive-testing", "")])
|
||||
|
||||
def test_revoke_with_reason(self):
|
||||
d = sop.parse_directives(
|
||||
"/sop-revoke comprehensive-testing realized the e2e was mocking the DB",
|
||||
self.aliases,
|
||||
)
|
||||
self.assertEqual(d[0][0], "sop-revoke")
|
||||
self.assertEqual(d[0][1], "comprehensive-testing")
|
||||
self.assertIn("mocking", d[0][2])
|
||||
|
||||
def test_directive_in_middle_of_comment(self):
|
||||
body = (
|
||||
"Reviewed the PR, looks good overall.\n"
|
||||
"/sop-ack comprehensive-testing\n"
|
||||
"Will follow up on the doc nit separately."
|
||||
)
|
||||
d = sop.parse_directives(body, self.aliases)
|
||||
self.assertEqual(len(d), 1)
|
||||
self.assertEqual(d[0][1], "comprehensive-testing")
|
||||
|
||||
def test_multiple_directives_in_one_comment(self):
|
||||
body = (
|
||||
"/sop-ack comprehensive-testing\n"
|
||||
"/sop-ack local-postgres-e2e\n"
|
||||
)
|
||||
d = sop.parse_directives(body, self.aliases)
|
||||
self.assertEqual(len(d), 2)
|
||||
slugs = {x[1] for x in d}
|
||||
self.assertEqual(slugs, {"comprehensive-testing", "local-postgres-e2e"})
|
||||
|
||||
def test_must_be_at_line_start(self):
|
||||
# A directive embedded mid-line is not honored (prevents review
|
||||
# comments like "to /sop-ack you need..." from acting as acks).
|
||||
body = "If you want to /sop-ack comprehensive-testing reply in this thread"
|
||||
d = sop.parse_directives(body, self.aliases)
|
||||
self.assertEqual(d, [])
|
||||
|
||||
def test_leading_whitespace_allowed(self):
|
||||
body = " /sop-ack comprehensive-testing"
|
||||
d = sop.parse_directives(body, self.aliases)
|
||||
self.assertEqual(len(d), 1)
|
||||
|
||||
def test_empty_body(self):
|
||||
self.assertEqual(sop.parse_directives("", self.aliases), [])
|
||||
self.assertEqual(sop.parse_directives(None, self.aliases), [])
|
||||
|
||||
def test_normalization_applied(self):
|
||||
# /sop-ack Comprehensive_Testing → canonical comprehensive-testing
|
||||
d = sop.parse_directives("/sop-ack Comprehensive_Testing", self.aliases)
|
||||
self.assertEqual(d[0][1], "comprehensive-testing")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# section_marker_present
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSectionMarkerPresent(unittest.TestCase):
|
||||
def test_marker_with_inline_answer(self):
|
||||
body = "- [ ] **Comprehensive testing performed**: Added 12 new tests covering null/empty/giant inputs."
|
||||
self.assertTrue(sop.section_marker_present(body, "Comprehensive testing performed"))
|
||||
|
||||
def test_marker_with_empty_answer(self):
|
||||
body = "- [ ] **Comprehensive testing performed**:"
|
||||
self.assertFalse(sop.section_marker_present(body, "Comprehensive testing performed"))
|
||||
|
||||
def test_marker_with_only_whitespace_answer(self):
|
||||
body = "- [ ] **Comprehensive testing performed**: \n"
|
||||
self.assertFalse(sop.section_marker_present(body, "Comprehensive testing performed"))
|
||||
|
||||
def test_marker_with_next_line_answer(self):
|
||||
body = (
|
||||
"- [ ] **Comprehensive testing performed**:\n"
|
||||
" Yes — see attached log + 12 new unit tests in foo_test.py.\n"
|
||||
)
|
||||
self.assertTrue(sop.section_marker_present(body, "Comprehensive testing performed"))
|
||||
|
||||
def test_marker_missing(self):
|
||||
body = "- [ ] **Local-postgres E2E run**: N/A — pure-frontend\n"
|
||||
self.assertFalse(sop.section_marker_present(body, "Comprehensive testing performed"))
|
||||
|
||||
def test_case_insensitive_marker_match(self):
|
||||
body = "- [ ] **comprehensive TESTING performed**: yes"
|
||||
self.assertTrue(sop.section_marker_present(body, "Comprehensive testing performed"))
|
||||
|
||||
def test_empty_body(self):
|
||||
self.assertFalse(sop.section_marker_present("", "X"))
|
||||
self.assertFalse(sop.section_marker_present(None, "X"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# compute_ack_state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestComputeAckState(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.items = _items_by_slug()
|
||||
self.aliases = _numeric_aliases()
|
||||
|
||||
@staticmethod
|
||||
def _approve_all(slug, users):
|
||||
return list(users)
|
||||
|
||||
@staticmethod
|
||||
def _approve_none(slug, users):
|
||||
return []
|
||||
|
||||
def _approve_only(self, allowed_users):
|
||||
return lambda slug, users: [u for u in users if u in allowed_users]
|
||||
|
||||
def test_peer_ack_passes(self):
|
||||
comments = [_comment("bob", "/sop-ack comprehensive-testing")]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], ["bob"])
|
||||
|
||||
def test_self_ack_rejected(self):
|
||||
comments = [_comment("alice", "/sop-ack comprehensive-testing")]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], [])
|
||||
self.assertEqual(state["comprehensive-testing"]["rejected"]["self_ack"], ["alice"])
|
||||
|
||||
def test_not_in_team_rejected(self):
|
||||
comments = [_comment("eve", "/sop-ack comprehensive-testing")]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_none
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], [])
|
||||
self.assertEqual(state["comprehensive-testing"]["rejected"]["not_in_team"], ["eve"])
|
||||
|
||||
def test_revoke_invalidates_own_prior_ack(self):
|
||||
# Bob acks then later revokes — Bob no longer counts.
|
||||
comments = [
|
||||
_comment("bob", "/sop-ack comprehensive-testing"),
|
||||
_comment("bob", "/sop-revoke comprehensive-testing realized e2e was mocked"),
|
||||
]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], [])
|
||||
|
||||
def test_revoke_does_not_affect_others_acks(self):
|
||||
# Bob revokes his own ack; Carol's still counts.
|
||||
comments = [
|
||||
_comment("bob", "/sop-ack comprehensive-testing"),
|
||||
_comment("carol", "/sop-ack comprehensive-testing"),
|
||||
_comment("bob", "/sop-revoke comprehensive-testing"),
|
||||
]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], ["carol"])
|
||||
|
||||
def test_ack_after_revoke_restored(self):
|
||||
# Bob revokes then re-acks (e.g. after re-reviewing).
|
||||
comments = [
|
||||
_comment("bob", "/sop-ack comprehensive-testing"),
|
||||
_comment("bob", "/sop-revoke comprehensive-testing"),
|
||||
_comment("bob", "/sop-ack comprehensive-testing"),
|
||||
]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], ["bob"])
|
||||
|
||||
def test_numeric_shorthand_ack(self):
|
||||
# /sop-ack 1 → comprehensive-testing
|
||||
comments = [_comment("bob", "/sop-ack 1")]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], ["bob"])
|
||||
|
||||
def test_ack_for_unknown_slug_ignored(self):
|
||||
# Some other slug not in config — silently drop (doesn't crash).
|
||||
comments = [_comment("bob", "/sop-ack does-not-exist")]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
for slug in self.items:
|
||||
self.assertEqual(state[slug]["ackers"], [])
|
||||
|
||||
def test_multi_item_multi_user(self):
|
||||
comments = [
|
||||
_comment("bob", "/sop-ack comprehensive-testing\n/sop-ack staging-smoke"),
|
||||
_comment("carol", "/sop-ack five-axis-review"),
|
||||
]
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", self.items, self.aliases, self._approve_all
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], ["bob"])
|
||||
self.assertEqual(state["staging-smoke"]["ackers"], ["bob"])
|
||||
self.assertEqual(state["five-axis-review"]["ackers"], ["carol"])
|
||||
self.assertEqual(state["root-cause"]["ackers"], [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# render_status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRenderStatus(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.items = _items()
|
||||
self.items_by_slug = _items_by_slug()
|
||||
|
||||
def _state_with(self, acked: list[str]) -> dict:
|
||||
return {
|
||||
it["slug"]: {
|
||||
"ackers": ["peer"] if it["slug"] in acked else [],
|
||||
"rejected": {"self_ack": [], "not_in_team": []},
|
||||
}
|
||||
for it in self.items
|
||||
}
|
||||
|
||||
def test_all_acked_returns_success(self):
|
||||
all_slugs = [it["slug"] for it in self.items]
|
||||
state, desc = sop.render_status(
|
||||
self.items, self._state_with(all_slugs), {s: True for s in all_slugs}
|
||||
)
|
||||
self.assertEqual(state, "success")
|
||||
self.assertIn("7/7", desc)
|
||||
|
||||
def test_partial_acked_returns_failure(self):
|
||||
state, desc = sop.render_status(
|
||||
self.items,
|
||||
self._state_with(["comprehensive-testing", "staging-smoke"]),
|
||||
{it["slug"]: True for it in self.items},
|
||||
)
|
||||
self.assertEqual(state, "failure")
|
||||
self.assertIn("2/7", desc)
|
||||
self.assertIn("missing", desc)
|
||||
|
||||
def test_description_truncates_long_missing_list(self):
|
||||
# Only ack one — 6 missing should be summarized as "+N".
|
||||
state, desc = sop.render_status(
|
||||
self.items,
|
||||
self._state_with(["comprehensive-testing"]),
|
||||
{it["slug"]: True for it in self.items},
|
||||
)
|
||||
# Length budget: under 140 chars.
|
||||
self.assertLessEqual(len(desc), 140)
|
||||
self.assertIn("+", desc) # +N elision marker
|
||||
|
||||
def test_body_unfilled_surfaced(self):
|
||||
all_slugs = [it["slug"] for it in self.items]
|
||||
state, desc = sop.render_status(
|
||||
self.items,
|
||||
self._state_with(all_slugs),
|
||||
{it["slug"]: False for it in self.items},
|
||||
)
|
||||
self.assertIn("body-unfilled", desc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_tier_mode
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetTierMode(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.cfg = sop.load_config(CONFIG_PATH)
|
||||
|
||||
def test_tier_high_is_hard(self):
|
||||
pr = {"labels": [{"name": "tier:high"}, {"name": "area:ci"}]}
|
||||
self.assertEqual(sop.get_tier_mode(pr, self.cfg), "hard")
|
||||
|
||||
def test_tier_medium_is_hard(self):
|
||||
pr = {"labels": [{"name": "tier:medium"}]}
|
||||
self.assertEqual(sop.get_tier_mode(pr, self.cfg), "hard")
|
||||
|
||||
def test_tier_low_is_soft(self):
|
||||
pr = {"labels": [{"name": "tier:low"}]}
|
||||
self.assertEqual(sop.get_tier_mode(pr, self.cfg), "soft")
|
||||
|
||||
def test_no_tier_label_defaults_to_hard(self):
|
||||
# Per feedback_fix_root_not_symptom — never silently lower the bar.
|
||||
pr = {"labels": [{"name": "area:ci"}]}
|
||||
self.assertEqual(sop.get_tier_mode(pr, self.cfg), "hard")
|
||||
|
||||
def test_no_labels_defaults_to_hard(self):
|
||||
self.assertEqual(sop.get_tier_mode({"labels": []}, self.cfg), "hard")
|
||||
self.assertEqual(sop.get_tier_mode({}, self.cfg), "hard")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# load_config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLoadConfig(unittest.TestCase):
|
||||
def test_default_config_parses(self):
|
||||
cfg = sop.load_config(CONFIG_PATH)
|
||||
self.assertIn("items", cfg)
|
||||
self.assertEqual(len(cfg["items"]), 7)
|
||||
slugs = {it["slug"] for it in cfg["items"]}
|
||||
self.assertEqual(
|
||||
slugs,
|
||||
{
|
||||
"comprehensive-testing",
|
||||
"local-postgres-e2e",
|
||||
"staging-smoke",
|
||||
"root-cause",
|
||||
"five-axis-review",
|
||||
"no-backwards-compat",
|
||||
"memory-consulted",
|
||||
},
|
||||
)
|
||||
|
||||
def test_default_config_tier_mode_shape(self):
|
||||
cfg = sop.load_config(CONFIG_PATH)
|
||||
self.assertEqual(cfg["tier_failure_mode"]["tier:high"], "hard")
|
||||
self.assertEqual(cfg["tier_failure_mode"]["tier:medium"], "hard")
|
||||
self.assertEqual(cfg["tier_failure_mode"]["tier:low"], "soft")
|
||||
self.assertEqual(cfg["default_mode"], "hard")
|
||||
|
||||
def test_each_item_has_required_fields(self):
|
||||
cfg = sop.load_config(CONFIG_PATH)
|
||||
for it in cfg["items"]:
|
||||
self.assertIn("slug", it)
|
||||
self.assertIn("numeric_alias", it)
|
||||
self.assertIn("pr_section_marker", it)
|
||||
self.assertIn("required_teams", it)
|
||||
self.assertIsInstance(it["required_teams"], list)
|
||||
self.assertGreater(len(it["required_teams"]), 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edge case: full integration without team probe (dependency-injected)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEndToEndAckFlow(unittest.TestCase):
|
||||
"""All-7-items happy path with synthetic comments. Verifies the
|
||||
full pipeline minus the Gitea API."""
|
||||
|
||||
def test_all_seven_acked_by_proper_teams(self):
|
||||
items = _items_by_slug()
|
||||
aliases = _numeric_aliases()
|
||||
comments = [
|
||||
_comment("qa-bot", "/sop-ack comprehensive-testing"),
|
||||
_comment("eng-bot", "/sop-ack local-postgres-e2e"),
|
||||
_comment("eng-bot", "/sop-ack staging-smoke"),
|
||||
_comment("mgr-bot", "/sop-ack root-cause"),
|
||||
_comment("eng-bot", "/sop-ack five-axis-review"),
|
||||
_comment("mgr-bot", "/sop-ack no-backwards-compat"),
|
||||
_comment("eng-bot", "/sop-ack memory-consulted"),
|
||||
]
|
||||
|
||||
def probe(slug, users):
|
||||
# Pretend every user is in every team.
|
||||
return list(users)
|
||||
|
||||
state = sop.compute_ack_state(comments, "alice-author", items, aliases, probe)
|
||||
body = {it["slug"]: True for it in items.values()}
|
||||
items_list = list(items.values())
|
||||
result_state, desc = sop.render_status(items_list, state, body)
|
||||
self.assertEqual(result_state, "success")
|
||||
self.assertIn("7/7", desc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -6,10 +6,9 @@
|
||||
# T1: PR open + APPROVED via tier:low → script invokes sop-tier-check
|
||||
# and POSTs status=success.
|
||||
# T2: PR open + missing tier label → sop-tier-check exits non-zero;
|
||||
# refire still POSTs status=success, matching the canonical
|
||||
# pull_request_target workflow's fail-open job conclusion.
|
||||
# refire POSTs status=failure (description mentions failure).
|
||||
# T3: PR open + tier:low but NO approving reviews → sop-tier-check
|
||||
# exits non-zero; refire still POSTs status=success for the same reason.
|
||||
# exits non-zero; refire POSTs status=failure.
|
||||
# T4: PR CLOSED → refire exits 0 with no status POST (no-op on closed).
|
||||
# T5: Rate-limit — recent status update within 30s → refire skips,
|
||||
# no new POST.
|
||||
@@ -33,7 +32,6 @@ THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT_DIR="$(cd "$THIS_DIR/.." && pwd)"
|
||||
WORKFLOW_DIR="$(cd "$THIS_DIR/../../workflows" && pwd)"
|
||||
WORKFLOW="$WORKFLOW_DIR/sop-tier-refire.yml"
|
||||
DISPATCH_WORKFLOW="$WORKFLOW_DIR/sop-checklist.yml"
|
||||
SCRIPT="$SCRIPT_DIR/sop-tier-refire.sh"
|
||||
|
||||
PASS=0
|
||||
@@ -89,7 +87,6 @@ assert_file_exists() {
|
||||
echo
|
||||
echo "== existence =="
|
||||
assert_file_exists "workflow file exists" "$WORKFLOW"
|
||||
assert_file_exists "SSOT dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
|
||||
assert_file_exists "script file exists" "$SCRIPT"
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo
|
||||
@@ -107,44 +104,30 @@ echo "== T6/T7 workflow yaml =="
|
||||
PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$WORKFLOW" 2>&1 || true)
|
||||
assert_eq "T7 workflow parses as YAML" "ok" "$PARSE_OUT"
|
||||
|
||||
# The old per-workflow issue_comment listener caused queue storms because
|
||||
# Gitea queues jobs before evaluating job-level `if:`. The script remains,
|
||||
# but comment-triggered refires route through the single dispatcher.
|
||||
# Three required gates in the `if:` expression
|
||||
WORKFLOW_CONTENT=$(cat "$WORKFLOW")
|
||||
if printf '%s' "$WORKFLOW_CONTENT" | grep -q '^ issue_comment:'; then
|
||||
echo " FAIL T6a manual fallback workflow must not listen on issue_comment"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T6a"
|
||||
else
|
||||
echo " PASS T6a manual fallback workflow does not listen on issue_comment"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
assert_contains "T6b workflow exposes workflow_dispatch" \
|
||||
"workflow_dispatch" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6c workflow documents unsupported manual inputs" \
|
||||
"workflow_dispatch inputs" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6a workflow if: contains author_association gate" \
|
||||
"github.event.comment.author_association" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6b workflow if: gates on MEMBER/OWNER/COLLABORATOR" \
|
||||
'["MEMBER","OWNER","COLLABORATOR"]' "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6c workflow if: contains slash-command trigger" \
|
||||
"/refire-tier-check" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6d workflow if: gates on PR-not-issue" \
|
||||
"github.event.issue.pull_request" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6e workflow listens on issue_comment" \
|
||||
"issue_comment" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6f workflow requests statuses:write permission" \
|
||||
"statuses: write" "$WORKFLOW_CONTENT"
|
||||
# Does NOT check out PR HEAD (security)
|
||||
if grep -q 'ref: \${{ github.event.pull_request.head' "$WORKFLOW"; then
|
||||
echo " FAIL T6d workflow MUST NOT check out PR head (security)"
|
||||
echo " FAIL T6g workflow MUST NOT check out PR head (security)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T6d"
|
||||
FAILED_TESTS="${FAILED_TESTS} T6g"
|
||||
else
|
||||
echo " PASS T6d workflow does not check out PR head"
|
||||
echo " PASS T6g workflow does not check out PR head"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
|
||||
DISPATCH_PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$DISPATCH_WORKFLOW" 2>&1 || true)
|
||||
assert_eq "T6e SSOT dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
|
||||
DISPATCH_CONTENT=$(cat "$DISPATCH_WORKFLOW")
|
||||
assert_contains "T6f SSOT dispatcher listens on issue_comment" \
|
||||
"issue_comment" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6g SSOT dispatcher handles /qa-recheck" \
|
||||
"/qa-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6h SSOT dispatcher handles /security-recheck" \
|
||||
"/security-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6i SSOT dispatcher handles /refire-tier-check" \
|
||||
"/refire-tier-check" "$DISPATCH_CONTENT"
|
||||
|
||||
# T1-T5 — script behavior against a local Gitea-fixture
|
||||
echo
|
||||
echo "== T1-T5 script behavior (vs local fixture) =="
|
||||
@@ -246,21 +229,34 @@ assert_contains "T1 POST context is sop-tier-check / tier-check" \
|
||||
'"context": "sop-tier-check / tier-check (pull_request)"' "$POSTED"
|
||||
assert_contains "T1 description names commenter" "test-runner" "$POSTED"
|
||||
|
||||
# T2: missing tier label → tier-check fails internally, but refire status
|
||||
# matches the canonical workflow's fail-open job conclusion.
|
||||
# T2: missing tier label → tier-check fails → failure status POSTed
|
||||
run_scenario "T2_no_tier_label" "fail_no_label"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T2 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T2 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
# tier-check.sh exits 1; refire script forwards that exit, so RC != 0
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T2 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T2 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T2_rc"
|
||||
fi
|
||||
assert_contains "T2 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T3: tier:low present but ZERO approving reviews → internal tier check fails,
|
||||
# refire status remains aligned with the canonical workflow.
|
||||
# T3: tier:low present but ZERO approving reviews → failure
|
||||
run_scenario "T3_no_approvals" "fail_no_approvals"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T3 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T3 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T3 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T3 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T3_rc"
|
||||
fi
|
||||
assert_contains "T3 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T4: closed PR — refire is a no-op (no POST, exit 0)
|
||||
run_scenario "T4_closed" "pass"
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import urllib.error
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "status-reaper.py"
|
||||
|
||||
|
||||
def load_reaper():
|
||||
spec = importlib.util.spec_from_file_location("status_reaper", SCRIPT)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(mod)
|
||||
mod.API = "https://git.example.test/api/v1"
|
||||
mod.GITEA_TOKEN = "fixture-token"
|
||||
mod.API_TIMEOUT_SEC = 1
|
||||
mod.API_RETRIES = 3
|
||||
mod.API_RETRY_SLEEP_SEC = 0
|
||||
return mod
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
status = 200
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps(self.payload).encode("utf-8")
|
||||
|
||||
|
||||
def test_api_retries_transient_timeout(monkeypatch):
|
||||
mod = load_reaper()
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_urlopen(req, timeout):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise TimeoutError("simulated slow Gitea API")
|
||||
return FakeResponse({"ok": True})
|
||||
|
||||
monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
status, body = mod.api("GET", "/repos/o/r/commits")
|
||||
|
||||
assert status == 200
|
||||
assert body == {"ok": True}
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_api_raises_after_retry_budget(monkeypatch):
|
||||
mod = load_reaper()
|
||||
|
||||
def fake_urlopen(req, timeout):
|
||||
raise urllib.error.URLError("connection reset")
|
||||
|
||||
monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
try:
|
||||
mod.api("GET", "/repos/o/r/commits")
|
||||
except mod.ApiError as exc:
|
||||
assert "failed after 3 attempts" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected ApiError")
|
||||
|
||||
|
||||
def test_reap_compensates_failed_pr_context_when_push_equivalent_passed(monkeypatch):
|
||||
mod = load_reaper()
|
||||
posted = []
|
||||
|
||||
def fake_post(sha, context, target_url, *, description="", dry_run=False):
|
||||
posted.append((sha, context, target_url, description, dry_run))
|
||||
|
||||
monkeypatch.setattr(mod, "post_compensating_status", fake_post)
|
||||
|
||||
counters = mod.reap(
|
||||
{"CI": True, "Handlers Postgres Integration": True},
|
||||
{
|
||||
"statuses": [
|
||||
{
|
||||
"context": "CI / Platform (Go) (pull_request)",
|
||||
"status": "failure",
|
||||
"target_url": "https://git.example.test/ci-pr",
|
||||
},
|
||||
{
|
||||
"context": "CI / Platform (Go) (push)",
|
||||
"status": "success",
|
||||
},
|
||||
{
|
||||
"context": (
|
||||
"Handlers Postgres Integration / "
|
||||
"Handlers Postgres Integration (pull_request)"
|
||||
),
|
||||
"status": "failure",
|
||||
"target_url": "https://git.example.test/handlers-pr",
|
||||
},
|
||||
{
|
||||
"context": (
|
||||
"Handlers Postgres Integration / "
|
||||
"Handlers Postgres Integration (push)"
|
||||
),
|
||||
"status": "success",
|
||||
},
|
||||
],
|
||||
},
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
)
|
||||
|
||||
assert counters["compensated_pr_shadowed_by_push_success"] == 2
|
||||
assert posted == [
|
||||
(
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
"CI / Platform (Go) (pull_request)",
|
||||
"https://git.example.test/ci-pr",
|
||||
mod.PR_SHADOW_COMPENSATION_DESCRIPTION,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
"Handlers Postgres Integration / Handlers Postgres Integration (pull_request)",
|
||||
"https://git.example.test/handlers-pr",
|
||||
mod.PR_SHADOW_COMPENSATION_DESCRIPTION,
|
||||
False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def test_reap_preserves_failed_pr_context_without_push_success(monkeypatch):
|
||||
mod = load_reaper()
|
||||
posted = []
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"post_compensating_status",
|
||||
lambda sha, context, target_url, *, description="", dry_run=False: posted.append(
|
||||
context
|
||||
),
|
||||
)
|
||||
|
||||
counters = mod.reap(
|
||||
{"CI": True},
|
||||
{
|
||||
"statuses": [
|
||||
{
|
||||
"context": "CI / Platform (Go) (pull_request)",
|
||||
"status": "failure",
|
||||
},
|
||||
{
|
||||
"context": "CI / Platform (Go) (push)",
|
||||
"status": "failure",
|
||||
},
|
||||
{
|
||||
"context": "CI / Shellcheck (pull_request)",
|
||||
"status": "failure",
|
||||
},
|
||||
],
|
||||
},
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
)
|
||||
|
||||
assert counters["preserved_pr_without_push_success"] == 2
|
||||
assert posted == []
|
||||
@@ -32,26 +32,6 @@
|
||||
# AUTHOR SELF-ACK IS FORBIDDEN regardless of which team contains them
|
||||
# — the gate script enforces commenter != PR author before checking
|
||||
# team membership.
|
||||
#
|
||||
# AI-SOP-ACK TEAM (internal#760 ceremony design, CTO-approved):
|
||||
# The `ai-sop-ack` team contains AI agent identities that can ack
|
||||
# SOP-checklist items ON BEHALF OF automated evidence. An AI ack is
|
||||
# only valid when:
|
||||
# 1. the item has `ai_ack_eligible: true`
|
||||
# 2. the item is NOT in the human-only carve-out (migration/schema)
|
||||
# 3. for testing-class items, CI / all-required (pull_request) is
|
||||
# green on the current head SHA
|
||||
#
|
||||
# AI acks NEVER count toward qa-review or security-review gates —
|
||||
# those remain human-team-only (enforced by review-check.sh team
|
||||
# probe against TEAM_ID 20/21).
|
||||
#
|
||||
# INITIAL ai_ack_eligible allowlist (CTO-controlled, msg 1388c76f):
|
||||
# comprehensive-testing, local-postgres-e2e, staging-smoke,
|
||||
# five-axis-review, memory-consulted
|
||||
# HUMAN-ONLY carve-out:
|
||||
# root-cause, no-backwards-compat
|
||||
# Any widening requires an explicit config change reviewed by CTO.
|
||||
|
||||
version: 1
|
||||
|
||||
@@ -70,83 +50,44 @@ tier_failure_mode:
|
||||
"tier:low": soft
|
||||
default_mode: hard # used when no tier:* label is present
|
||||
|
||||
# High-risk class (RFC#450 Option C, governance-fix for internal#442).
|
||||
#
|
||||
# A PR is "high-risk" when ANY of the listed labels are applied OR when
|
||||
# the PR has `tier:high` (mechanically the strictest existing tier).
|
||||
# High-risk items use `required_teams_high_risk` (when present on the
|
||||
# item); non-high-risk items use the default `required_teams`.
|
||||
#
|
||||
# This closes the inconsistency that the SOP charter already mandates
|
||||
# `tier:high → ceo only` for the sibling `sop-tier-check` gate; the
|
||||
# sop-checklist's `root-cause` and `no-backwards-compat` items now
|
||||
# follow the same risk-classed two-eyes shape:
|
||||
# - Default class (tier:low/medium, not high-risk): a non-author
|
||||
# engineers/managers/ceo ack satisfies the item — 25+ live
|
||||
# identities, no dependency on a dead/inactive senior persona
|
||||
# token.
|
||||
# - High-risk class (tier:high OR any high_risk_label): still
|
||||
# requires a non-author ceo ack (durable human team).
|
||||
#
|
||||
# Tightening: add labels to high_risk_labels.
|
||||
# Loosening: remove labels.
|
||||
high_risk_labels:
|
||||
- "risk:high"
|
||||
- "area:security"
|
||||
- "area:schema"
|
||||
- "area:fleet-image"
|
||||
- "area:identity"
|
||||
- "area:gate-meta"
|
||||
|
||||
items:
|
||||
- slug: comprehensive-testing
|
||||
numeric_alias: 1
|
||||
pr_section_marker: "Comprehensive testing performed"
|
||||
required_teams: [qa, engineers]
|
||||
ai_ack_eligible: true
|
||||
description: >-
|
||||
What was tested, how, edge cases covered. Ack from any qa-team
|
||||
member (or engineers fallback while qa is small). AI ack valid
|
||||
only when CI / all-required (pull_request) is green.
|
||||
member (or engineers fallback while qa is small).
|
||||
|
||||
- slug: local-postgres-e2e
|
||||
numeric_alias: 2
|
||||
pr_section_marker: "Local-postgres E2E run"
|
||||
required_teams: [engineers]
|
||||
ai_ack_eligible: true
|
||||
description: >-
|
||||
Link to local CI artifact, or "N/A: pure-frontend change". Ack
|
||||
from any engineer who can verify the local DB test actually ran.
|
||||
AI ack valid only when CI / all-required (pull_request) is green.
|
||||
|
||||
- slug: staging-smoke
|
||||
numeric_alias: 3
|
||||
pr_section_marker: "Staging-smoke verified or pending"
|
||||
required_teams: [engineers]
|
||||
ai_ack_eligible: true
|
||||
description: >-
|
||||
Link to canary run, or "scheduled post-merge". Ack from any
|
||||
engineer (core-devops/infra-sre are members of engineers team).
|
||||
AI ack valid only when CI / all-required (pull_request) is green.
|
||||
|
||||
- slug: root-cause
|
||||
numeric_alias: 4
|
||||
pr_section_marker: "Root-cause not symptom"
|
||||
required_teams: [engineers, managers, ceo]
|
||||
required_teams_high_risk: [ceo]
|
||||
required_teams: [managers, ceo]
|
||||
description: >-
|
||||
One-sentence root-cause statement. Default class: non-author
|
||||
engineers/managers/ceo ack suffices (engineers can attest
|
||||
root-cause-vs-symptom for routine fixes). High-risk class
|
||||
(see `high_risk_labels`): non-author ceo ack required —
|
||||
senior judgment for irreversible/security/identity/gate
|
||||
changes. Closes internal#442 + tracks RFC#450.
|
||||
One-sentence root-cause statement. Ack from managers tier
|
||||
(team-leads) or ceo. Senior judgment required to attest
|
||||
root-cause-versus-symptom.
|
||||
|
||||
- slug: five-axis-review
|
||||
numeric_alias: 5
|
||||
pr_section_marker: "Five-Axis review walked"
|
||||
required_teams: [engineers]
|
||||
ai_ack_eligible: true
|
||||
description: >-
|
||||
Correctness / readability / architecture / security / performance.
|
||||
Ack from any non-author engineer.
|
||||
@@ -154,56 +95,15 @@ items:
|
||||
- slug: no-backwards-compat
|
||||
numeric_alias: 6
|
||||
pr_section_marker: "No backwards-compat shim / dead code added"
|
||||
required_teams: [engineers, managers, ceo]
|
||||
required_teams_high_risk: [ceo]
|
||||
required_teams: [managers, ceo]
|
||||
description: >-
|
||||
Yes/no + justification if no. Default class: non-author
|
||||
engineers/managers/ceo ack suffices. High-risk class
|
||||
(see `high_risk_labels`): non-author ceo ack required —
|
||||
senior judgment for shim-versus-real-fix on irreversible
|
||||
surfaces. Closes internal#442 + tracks RFC#450.
|
||||
Yes/no + justification if no. Senior ack required because
|
||||
backward-compat shims are how dead-code accretes.
|
||||
|
||||
- slug: memory-consulted
|
||||
numeric_alias: 7
|
||||
pr_section_marker: "Memory/saved-feedback consulted"
|
||||
required_teams: [engineers]
|
||||
ai_ack_eligible: true
|
||||
description: >-
|
||||
List of feedback memories applicable to this change. Ack from
|
||||
any engineer who has the same memory access.
|
||||
|
||||
# N/A gate declarations (RFC#324 §N/A follow-up).
|
||||
# PRs where a gate genuinely does not apply (e.g., pure-infra with no
|
||||
# qa surface, or docs-only) can be declared N/A by a non-author peer
|
||||
# who is in one of the gate's required_teams. The sop-checklist
|
||||
# posts a `sop-checklist / na-declarations (pull_request)` status that
|
||||
# review-check.sh reads to skip the Gitea-APPROVE requirement.
|
||||
#
|
||||
# Usage: any PR commenter (peer) posts:
|
||||
# /sop-n/a qa-review <reason>
|
||||
# /sop-n/a security-review <reason>
|
||||
#
|
||||
# Slash commands:
|
||||
# /sop-n/a <gate> [reason] — declare gate N/A (most-recent per-user wins)
|
||||
# /sop-revoke <gate> — revoke prior N/A declaration for that gate
|
||||
#
|
||||
# Gate names must match the context strings used by review-check.sh:
|
||||
# qa-review → qa-review / approved (<event>) [TEAM_ID=20]
|
||||
# security-review → security-review / approved (<event>) [TEAM_ID=21]
|
||||
#
|
||||
# required_teams: OR semantics — any team member can declare N/A.
|
||||
# Authors cannot self-declare N/A (enforced by gate script).
|
||||
n/a_gates:
|
||||
qa-review:
|
||||
required_teams: [qa, security, engineers]
|
||||
description: >-
|
||||
QA review N/A when this change has no qa surface (pure-infra,
|
||||
tooling-only, revert, dependency-only). A qa/eng/security member
|
||||
must post /sop-n/a qa-review to activate.
|
||||
|
||||
security-review:
|
||||
required_teams: [security, managers, ceo]
|
||||
description: >-
|
||||
Security review N/A when this change has no security surface
|
||||
(docs-only, pure-frontend, dependency-only). A security/owners
|
||||
member must post /sop-n/a security-review to activate.
|
||||
|
||||
@@ -1,71 +1,89 @@
|
||||
# audit-force-merge — emit `incident.force_merge` to runner stdout when
|
||||
# a PR is merged with required-status-checks not green. Vector picks
|
||||
# audit-force-merge — emit `incident.force_merge` to the runner log when
|
||||
# a PR is merged with required-status checks NOT all green. Vector picks
|
||||
# the JSON line off docker_logs and ships to Loki on
|
||||
# molecule-canonical-obs (per `reference_obs_stack_phase1`); query as:
|
||||
#
|
||||
# {host="operator"} |= "event_type" |= "incident.force_merge" | json
|
||||
#
|
||||
# Closes the §SOP-6 audit gap (the doc says force-merges write to
|
||||
# `structure_events`, but that table lives in the platform DB, not
|
||||
# Gitea-side; Loki is the practical equivalent for Gitea Actions
|
||||
# events). When the credential / observability stack converges later,
|
||||
# this can sync into structure_events from Loki via a backfill job —
|
||||
# the structured JSON shape is forward-compatible.
|
||||
# Companion to `audit-force-merge.sh` (script-extract pattern, same as
|
||||
# sop-tier-check). The audit observes BOTH UI-merged and REST-merged PRs
|
||||
# uniformly per `feedback_gh_cli_merge_lies_use_rest`.
|
||||
#
|
||||
# Logic in `.gitea/scripts/audit-force-merge.sh` per the same script-
|
||||
# extract pattern as sop-tier-check.
|
||||
# Closes the §SOP-6 audit gap for the molecule-core repo. RFC:
|
||||
# internal#219 §6. Mirrors the same-named workflow in
|
||||
# molecule-controlplane; design rationale lives in the RFC, not here,
|
||||
# to keep the workflow file scannable.
|
||||
|
||||
name: audit-force-merge
|
||||
|
||||
# pull_request_target loads from the base branch — same security model
|
||||
# as sop-tier-check. Without this, an attacker could rewrite the
|
||||
# workflow on a PR and skip the audit emission for their own
|
||||
# force-merge. See `.gitea/workflows/sop-tier-check.yml` for the full
|
||||
# rationale.
|
||||
# as sop-tier-check. Without this, a PR author could rewrite the
|
||||
# workflow on their own PR and skip the audit emission for their own
|
||||
# force-merge. The base-branch checkout below ALSO uses
|
||||
# `base.sha`, not `base.ref`, so a fast-moving base can't slip a
|
||||
# different audit script in under us.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
|
||||
# `pull-requests: read` + `contents: read` covers everything the script
|
||||
# needs (fetch PR + commit statuses). `issues:` deliberately omitted —
|
||||
# audit fires-and-forgets to stdout, never opens issues.
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
# Skip when PR is closed without merge — saves a runner.
|
||||
if: github.event.pull_request.merged == true
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# base.sha pinning, NOT base.ref — see header rationale.
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
- name: Detect force-merge + emit audit event
|
||||
env:
|
||||
# Same org-level secret the sop-tier-check workflow uses.
|
||||
# Same org-level secret the sop-tier-check workflow uses;
|
||||
# falls back to the auto-injected GITHUB_TOKEN if the
|
||||
# org-level SOP_TIER_CHECK_TOKEN isn't set on a transitional
|
||||
# repo.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
# Required-status-check contexts to evaluate at merge time.
|
||||
# Branch-aware JSON dict: keys are protected branch names,
|
||||
# values are arrays of context names that branch protection
|
||||
# requires for that branch. Mirror this against branch
|
||||
# protection (settings → branches → protected branch →
|
||||
# required checks) for each branch listed here.
|
||||
# Newline-separated. MUST mirror branch protection's
|
||||
# status_check_contexts for protected branches
|
||||
# (currently `main`; `staging` protection forthcoming per
|
||||
# RFC internal#219 Phase 4).
|
||||
#
|
||||
# Initialized 2026-05-11 from the current molecule-core `main`
|
||||
# branch protection:
|
||||
#
|
||||
# GET /api/v1/repos/molecule-ai/molecule-core/
|
||||
# branch_protections/main
|
||||
# → status_check_contexts = [
|
||||
# "Secret scan / Scan diff for credential-shaped strings (pull_request)",
|
||||
# "sop-tier-check / tier-check (pull_request)"
|
||||
# ]
|
||||
#
|
||||
# Declared here rather than fetched from /branch_protections
|
||||
# because that endpoint requires admin write — sop-tier-bot is
|
||||
# read-only by design (least-privilege).
|
||||
REQUIRED_CHECKS_JSON: |
|
||||
{
|
||||
"main": [
|
||||
"CI / all-required (pull_request)",
|
||||
"E2E API Smoke Test / E2E API Smoke Test (pull_request)",
|
||||
"Handlers Postgres Integration / Handlers Postgres Integration (pull_request)"
|
||||
],
|
||||
"staging": [
|
||||
"CI / all-required (pull_request)",
|
||||
"sop-checklist / all-items-acked (pull_request)"
|
||||
]
|
||||
}
|
||||
# because that endpoint requires admin write — sop-tier-bot
|
||||
# is read-only by design (least-privilege per
|
||||
# `feedback_least_privilege_via_workflow_env` / internal#257).
|
||||
# Drift between this env and the real protection list is
|
||||
# auto-detected by `ci-required-drift.yml` (RFC §4 + §6),
|
||||
# which opens a `[ci-drift]` issue within one hour.
|
||||
#
|
||||
# When the protection set changes (e.g. Phase 4 adds the
|
||||
# `ci / all-required (pull_request)` sentinel), update BOTH
|
||||
# branch protection AND this env in the SAME PR; drift-detect
|
||||
# will otherwise file an issue for you.
|
||||
REQUIRED_CHECKS: |
|
||||
Secret scan / Scan diff for credential-shaped strings (pull_request)
|
||||
sop-tier-check / tier-check (pull_request)
|
||||
CI / all-required (pull_request)
|
||||
run: bash .gitea/scripts/audit-force-merge.sh
|
||||
|
||||
@@ -37,7 +37,6 @@ jobs:
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Ported from .github/workflows/cascade-list-drift-gate.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths reference .gitea/workflows/publish-runtime.yml (the active
|
||||
# Gitea workflow file) instead of .github/workflows/publish-runtime.yml
|
||||
# (which Category A of this sweep deletes).
|
||||
# - Explicit `WORKFLOW=` arg passed to the drift script so it audits the
|
||||
# .gitea/ workflow (the script's default is still .github/... which
|
||||
# will not exist post-Cat-A).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract — surface
|
||||
# defects without blocking; follow-up PR flips after triage).
|
||||
#
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .gitea/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
# Pass the .gitea/ workflow path explicitly — the script's
|
||||
# default still points at .github/... which Category A of this
|
||||
# sweep removes.
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh manifest.json .gitea/workflows/publish-runtime.yml
|
||||
@@ -42,9 +42,10 @@ jobs:
|
||||
check:
|
||||
name: Migration version collision check
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 4 (RFC #219 §1): 22 days green since 2026-05-11 port.
|
||||
# mc#1982 mask removed — no surfaced defects in this lane.
|
||||
continue-on-error: false
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
# ci-arm64-advisory — Mac arm64 self-hosted ADVISORY fast-check lane.
|
||||
#
|
||||
# === WHY ===
|
||||
#
|
||||
# The amd64 Gitea runner pool (molecule-runner-1..20) is queue-contended
|
||||
# (internal#418). This lane offloads the *genuinely container-independent*
|
||||
# fast checks (Go build/vet/lint, shellcheck, Python lint) onto the Mac
|
||||
# arm64 self-hosted runner so developers get a fast arm64 signal WITHOUT
|
||||
# adding load to the starved amd64 pool — capability-honestly, as an
|
||||
# additive pilot. Pilot ② of the Mac-CI strategy (CTO-delegated 2026-05-17).
|
||||
#
|
||||
# === NON-NEGOTIABLE SAFETY CONTRACT (the prime directive) ===
|
||||
#
|
||||
# This lane is **ADVISORY ONLY**. It is provably incapable of hanging a
|
||||
# merge. Concretely:
|
||||
#
|
||||
# 1. It is a SEPARATE workflow file. `ci.yml` is byte-for-byte
|
||||
# untouched by this PR. The `CI / all-required` aggregator sentinel
|
||||
# and the five contexts it polls
|
||||
# (`CI / Detect changes|Platform (Go)|Canvas (Next.js)|
|
||||
# Shellcheck (E2E scripts)|Python Lint & Test (pull_request)`)
|
||||
# are unchanged. The canonical required gate stays 100% on the
|
||||
# existing amd64 pool.
|
||||
#
|
||||
# 2. The context this workflow emits is
|
||||
# `ci-arm64-advisory / fast-checks (pull_request)`. That string is
|
||||
# DELIBERATELY NOT present in, and this PR does NOT add it to:
|
||||
# - branch_protections/{main,staging}.status_check_contexts
|
||||
# (DB-verified pb 86/75 = exactly
|
||||
# ["CI / all-required (pull_request)",
|
||||
# "sop-checklist / all-items-acked (pull_request)"])
|
||||
# - audit-force-merge.yml REQUIRED_CHECKS env
|
||||
# - ci.yml `all-required` sentinel's hardcoded `required[]` list
|
||||
# Branch protection therefore never waits on this context. If the
|
||||
# Mac runner is absent / offline / removed, this workflow's status
|
||||
# simply never appears — and because nothing requires it, every
|
||||
# merge proceeds exactly as it does today. There is no path by
|
||||
# which a missing/red arm64 status blocks a merge.
|
||||
#
|
||||
# 3. `continue-on-error: true` on the job — even a genuine arm64-only
|
||||
# failure (toolchain drift, arch-specific test flake) is surfaced
|
||||
# as information, never as a merge blocker, for the duration of
|
||||
# the pilot.
|
||||
#
|
||||
# 4. The job carries a `github.event_name` `if:` gate. Beyond its
|
||||
# functional purpose this also keeps the job OUT of
|
||||
# `ci-required-drift.py:ci_job_names()` (which excludes
|
||||
# `github.event_name`/`github.ref`-gated jobs), so the hourly
|
||||
# ci-required-drift sentinel's F1 ("job not under sentinel needs")
|
||||
# cannot ever flag this advisory job. F2/F3 are untouched because
|
||||
# this context is absent from BP and from REQUIRED_CHECKS.
|
||||
# `lint-bp-context-emit-match` only fails on BP→emitter gaps; an
|
||||
# emitter without a BP context is explicitly informational there.
|
||||
#
|
||||
# === RUNNER TARGETING ===
|
||||
#
|
||||
# The Mac runner is `hongming-pc-runner-1`. The bare `self-hosted`
|
||||
# label is POLLUTED in this Gitea instance: molecule-runner-1..20
|
||||
# (the contended amd64 pool) also advertise `self-hosted`. Targeting
|
||||
# bare `self-hosted` would route back onto the very pool we are trying
|
||||
# to relieve — and onto amd64 hardware. We therefore require an
|
||||
# AND-set of labels that ONLY the Mac satisfies. `macos-self-hosted`
|
||||
# is Mac-exclusive (the amd64 pool does not carry it). Until the
|
||||
# label-install burst (a10862b2) lands `self-hosted`+`macos-self-hosted`
|
||||
# on the Mac, the runner's current unique label `hongming-pc-laptop`
|
||||
# is also listed; AND-semantics over the labels a runner advertises
|
||||
# means a job requiring [self-hosted, macos-self-hosted] can ONLY be
|
||||
# claimed once the Mac advertises both. If neither label set is yet
|
||||
# present on the Mac, the workflow stays queued harmlessly and is
|
||||
# garbage-collected by the normal stale-run reaper — it blocks nothing
|
||||
# (see safety contract point 2).
|
||||
#
|
||||
# === ROLLBACK ===
|
||||
#
|
||||
# Delete this single file (`git rm .gitea/workflows/ci-arm64-advisory.yml`)
|
||||
# and merge. No branch-protection edit, no ci.yml edit, no
|
||||
# REQUIRED_CHECKS edit is required to roll back, because none were made
|
||||
# to roll forward. Zero blast radius either direction.
|
||||
|
||||
name: ci-arm64-advisory
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
# Per-ref cancel: a newer commit on the same ref supersedes the older
|
||||
# advisory run. Distinct from ci.yml's `ci-${ref}` group so this lane
|
||||
# never cancels (or is cancelled by) the canonical required CI.
|
||||
concurrency:
|
||||
group: ci-arm64-advisory-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
fast-checks:
|
||||
name: fast-checks
|
||||
# AND-set: only the Mac arm64 runner advertises macos-self-hosted.
|
||||
# See "RUNNER TARGETING" header note for why bare self-hosted is unsafe.
|
||||
runs-on: [self-hosted, macos-self-hosted]
|
||||
# ADVISORY: never blocks. See safety contract point 3. mc#1982
|
||||
# internal#418 — tracked: arm64 advisory pilot, non-gating by design.
|
||||
continue-on-error: true
|
||||
# event_name gate: functional (only meaningful on push/PR) AND keeps
|
||||
# this job out of ci-required-drift.py:ci_job_names() so F1 can never
|
||||
# flag it. See safety contract point 4.
|
||||
if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Provenance — advisory lane, non-gating
|
||||
run: |
|
||||
echo "This is the arm64 ADVISORY fast-check lane."
|
||||
echo "It does NOT gate merges. Canonical required CI is ci.yml"
|
||||
echo "on the amd64 pool. Arch: $(uname -m) on $(uname -s)."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# ---- Go: build + vet + lint (container-independent: needs only the
|
||||
# Go toolchain; no amd64 ECR image, no docker-in-job). Race-detector
|
||||
# unit-test + coverage gates are deliberately NOT duplicated here —
|
||||
# those stay authoritative on amd64 ci.yml `Platform (Go)`. This lane
|
||||
# is fast-feedback for the compile/vet/lint surface only. ----
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- name: Go build + vet (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go mod download
|
||||
go build ./cmd/server
|
||||
go vet ./...
|
||||
- name: golangci-lint (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
"$(go env GOPATH)/bin/golangci-lint" run --timeout 3m ./...
|
||||
|
||||
# ---- Shellcheck (container-independent: shellcheck binary only).
|
||||
# Mirrors ci.yml `Shellcheck (E2E scripts)` bulk pass scope. ----
|
||||
- name: Install shellcheck (arm64)
|
||||
run: |
|
||||
if ! command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "shellcheck not preinstalled on this self-hosted runner."
|
||||
echo "Attempting Homebrew install (Mac arm64)."
|
||||
brew install shellcheck || {
|
||||
echo "::warning::shellcheck unavailable on runner; advisory shellcheck skipped."
|
||||
exit 0
|
||||
}
|
||||
fi
|
||||
shellcheck --version
|
||||
- name: Shellcheck tests/e2e + infra/scripts
|
||||
run: |
|
||||
command -v shellcheck >/dev/null 2>&1 || { echo "skip"; exit 0; }
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
# ---- Python lint/compile (container-independent: CPython only).
|
||||
# Lint + import-compile surface; the authoritative pytest + coverage
|
||||
# floors stay on amd64 ci.yml `Python Lint & Test`. ----
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Python byte-compile (workspace)
|
||||
working-directory: workspace
|
||||
run: |
|
||||
python -m pip install --quiet ruff || true
|
||||
python -m compileall -q .
|
||||
if command -v ruff >/dev/null 2>&1; then
|
||||
ruff check . || echo "::warning::ruff findings (advisory only)"
|
||||
fi
|
||||
|
||||
- name: Advisory summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## arm64 advisory fast-checks complete"
|
||||
echo ""
|
||||
echo "This lane is **advisory** — it does not gate merges."
|
||||
echo "Authoritative required CI remains \`CI / all-required\`"
|
||||
echo "on the amd64 pool (\`ci.yml\`, unchanged by this PR)."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
+226
-223
@@ -86,68 +86,93 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
env:
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile ci \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "$PR_BASE_SHA" \
|
||||
--base-ref "$PR_BASE_REF" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-$PUSH_BEFORE}"
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# Both .github/workflows/ci.yml AND .gitea/workflows/ci.yml count
|
||||
# as "this workflow changed" — either edit should force-run every
|
||||
# downstream job. The Gitea port follows the same shape as the
|
||||
# GitHub original so behavior matches when triggered on either
|
||||
# platform.
|
||||
DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".gitea/workflows/ci.yml")
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The job always
|
||||
# emits the required context, but expensive steps are path-scoped on every
|
||||
# event so docs/E2E/Canvas-only main pushes do not block deploy on unrelated
|
||||
# Go bootstrap work.
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
|
||||
# + per-step gating shape preserves the GitHub-side required-check name
|
||||
# contract (so when this Gitea port becomes a required check in Phase 4,
|
||||
# the name match works on PRs that don't touch workspace-server/).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
# mc#1982 (closed 2026-05-14): Phase 4 flip of the platform-build job.
|
||||
# Phase 4 (#656) originally flipped this to continue-on-error: false based on
|
||||
# Phase-3-masked "green on main 2026-05-12". Two failure classes then surfaced:
|
||||
# (1) 4x delegation_test.go sqlmock gaps (PR #669 / #634 fix-forward, closed).
|
||||
# (2) TestMCPHandler_CommitMemory_GlobalScope_Blocked (mcp_test.go:433):
|
||||
# OFFSEC-001 hardening collided with test assertion; tracked in mc#762.
|
||||
# Fix-forward for (1) landed in PR #669. The mc#762 gap (2) is a separate
|
||||
# issue — it does NOT block this flip because the test is already wrapped in
|
||||
# the diagnostic step with its own continue-on-error: true (line 203).
|
||||
# Flip confirmed by CI / Platform (Go) status = success on main HEAD 363905d3.
|
||||
continue-on-error: false
|
||||
# Job-level ceiling. The go test step below runs with a per-step 10m timeout;
|
||||
# this cap catches any step that leaks past that. Set well above 10m so
|
||||
# the per-step timeout is the active constraint.
|
||||
timeout-minutes: 15
|
||||
# mc#664 (interim): re-mask platform-build pending fix-forward. Phase 4
|
||||
# (#656) flipped this to continue-on-error: false based on a Phase-3-masked
|
||||
# "green on main 2026-05-12" — the prior continue-on-error: true had
|
||||
# been hiding failing tests in workspace-server/internal/handlers/.
|
||||
# Two distinct failure classes surfaced on 0e5152c3:
|
||||
# (1) 4x delegation_test.go (lines 1110/1176/1228/1271): helpers
|
||||
# expectExecuteDelegationBase/Success/Failed are missing sqlmock
|
||||
# expectations for queries production has issued since ~2026-04-21
|
||||
# (last_outbound_at UPDATE, lookupDeliveryMode/Runtime SELECTs,
|
||||
# a2a_receive INSERT activity_logs, recordLedgerStatus writes).
|
||||
# Halt cond #3 applies (regression > 7 days → broader sweep).
|
||||
# (2) 1x mcp_test.go:433 (TestMCPHandler_CommitMemory_GlobalScope_Blocked):
|
||||
# commit 7d1a189f (2026-05-10) hardened mcp.go to scrub err.Error()
|
||||
# from JSON-RPC responses (OFFSEC-001), but the test asserts the
|
||||
# error message contains "GLOBAL". Production-vs-test contract
|
||||
# collision — needs design call, not mock update.
|
||||
# Time-boxed Option A (90 min) did not fit the cross-cutting scope.
|
||||
# This is a sequenced revert→fix→reflip per
|
||||
# feedback_strict_root_only_after_class_a emergency clause — NOT
|
||||
# a permanent re-mask. Re-flip blocked on mc#664 fix-forward landing.
|
||||
# Other 4 #656 flips (changes, canvas-build, shellcheck, python-lint)
|
||||
# retain continue-on-error: false; only platform-build regresses.
|
||||
continue-on-error: true # mc#664 fix-forward in flight; re-flip when tests pass
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.platform != 'true' }}
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace-server/** changes — Platform (Go) gate satisfied without running Go build/test/lint."
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: go vet ./...
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Install golangci-lint
|
||||
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: $(go env GOPATH)/bin/golangci-lint run --timeout 3m ./...
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Diagnostic — per-package verbose 60s
|
||||
run: |
|
||||
set +e
|
||||
@@ -161,25 +186,12 @@ jobs:
|
||||
echo "::group::pendinguploads exit=$pu_exit (last 100 lines)"
|
||||
tail -100 /tmp/test-pu.log
|
||||
echo "::endgroup::"
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Run tests with coverage (blocking gate)
|
||||
# Removed -race from the blocking gate per #1184: cold runners
|
||||
# take 13-25 min to compile with race instrumentation, exceeding
|
||||
# the 10m step timeout and causing false failures. Race detection
|
||||
# now runs as a non-blocking advisory step below.
|
||||
run: go test -timeout 10m -coverprofile=coverage.out ./...
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Race detection (advisory, non-blocking)
|
||||
# mc#1184: runs race detector as an advisory check so cold-runner
|
||||
# compile-time spikes don't block merges. Failures here surface in
|
||||
# the run log but do not fail the build.
|
||||
run: go test -race -timeout 10m ./...
|
||||
continue-on-error: true
|
||||
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
@@ -193,7 +205,7 @@ jobs:
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
@@ -247,7 +259,7 @@ jobs:
|
||||
# Strip the package-import prefix so we can match .coverage-allowlist.txt
|
||||
# entries written as paths relative to workspace-server/.
|
||||
# Handle both module paths: platform/workspace-server/... and platform/...
|
||||
rel=$(echo "$file" | sed 's|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/workspace-server/||; s|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/||')
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
|
||||
@@ -283,27 +295,26 @@ jobs:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.canvas != 'true' }}
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — Canvas (Next.js) gate satisfied without running npm build/test."
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: npm ci --include=optional --prefer-offline
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
@@ -312,7 +323,7 @@ jobs:
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
@@ -326,7 +337,7 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# Shellcheck (E2E scripts) — required context, path-scoped heavy steps.
|
||||
# Shellcheck (E2E scripts) — required check, always runs.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
@@ -334,11 +345,11 @@ jobs:
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.scripts != 'true' }}
|
||||
run: echo "No tests/e2e, scripts, or infra/scripts changes — Shellcheck gate satisfied without running script checks."
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
@@ -349,74 +360,31 @@ jobs:
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
# molecule-core#1995 (#1994 follow-on): fail-direction proof for
|
||||
# the A2A real-completion + byok-routing assertion helpers
|
||||
# (lib/completion_assert.sh). Offline (no LLM, no network): it
|
||||
# asserts an error-as-text payload FAILS the real-completion gate
|
||||
# — the exact trap the historical shape-only `"kind":"text"`
|
||||
# check missed. If a refactor weakens the gate to a shape check,
|
||||
# this step goes red on every PR.
|
||||
bash tests/e2e/test_completion_assert_unit.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Test ECR promote-tenant-image script (mock-driven, no live infra)
|
||||
# Covers scripts/promote-tenant-image.sh — the codified
|
||||
# :staging-latest → :latest ECR promote + tenant fleet redeploy
|
||||
# closing molecule-ai/molecule-core#660. 40 mock-driven cases
|
||||
# exercise every exit path (preflight, snapshot, promote, redeploy
|
||||
# 403→SSM-refresh, verify, rollback). No live AWS/CP/SSM calls.
|
||||
run: |
|
||||
bash scripts/test-promote-tenant-image.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Shellcheck promote-tenant-image script
|
||||
# scripts/ is excluded from the bulk shellcheck pass above (legacy
|
||||
# SC3040/SC3043 cleanup pending). Run shellcheck explicitly on
|
||||
# the promote script + its test harness so regressions there are
|
||||
# caught by the required check.
|
||||
run: |
|
||||
shellcheck --severity=warning \
|
||||
scripts/promote-tenant-image.sh \
|
||||
scripts/test-promote-tenant-image.sh
|
||||
|
||||
# mc#959 root-fix (sre)
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: docker-host
|
||||
# mc#1982 root-fix: added job-level `if:` so ci-required-drift.py's
|
||||
# ci_job_names() detects this as github.ref-gated and skips it from F1.
|
||||
# The step-level exit 0 handles the "not main push" case; the job-level
|
||||
# `if:` makes the gating explicit so the drift script sees it.
|
||||
# Runs on both main and staging pushes; step exits 0 when not applicable.
|
||||
if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' }}
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
env:
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
CANVAS_CHANGED: ${{ needs.changes.outputs.canvas }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
REF_NAME: ${{ github.ref }}
|
||||
# github.server_url resolves via the workflow-level env override
|
||||
# to the Gitea instance, so the RUN_URL points at the Gitea run
|
||||
# page (not github.com). See feedback_act_runner_github_server_url.
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$CANVAS_CHANGED" != "true" ] || [ "$EVENT_NAME" != "push" ] || [ "$REF_NAME" != "refs/heads/main" ]; then
|
||||
echo "Canvas deploy reminder not applicable for event=$EVENT_NAME ref=$REF_NAME canvas_changed=$CANVAS_CHANGED."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Write body to a temp file — avoids backtick escaping in shell.
|
||||
cat > /tmp/deploy-reminder.md << 'BODY'
|
||||
## Canvas build passed — deploy required
|
||||
@@ -446,40 +414,94 @@ jobs:
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs.
|
||||
# Runtime Python moved to molecule-ai-workspace-runtime. Keep this context as
|
||||
# a guard so branch protection still catches attempts to reintroduce an
|
||||
# editable runtime copy under molecule-core/workspace/.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Runtime SSOT guard
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. See issue #2790 for full rationale.
|
||||
run: |
|
||||
set -eu
|
||||
if [ -d workspace ]; then
|
||||
echo "::error file=workspace::Runtime source must live in molecule-ai-workspace-runtime, not molecule-core/workspace."
|
||||
exit 1
|
||||
fi
|
||||
for f in scripts/build_runtime_package.py scripts/test_build_runtime_package.py; do
|
||||
if [ -e "$f" ]; then
|
||||
echo "::error file=$f::Legacy build-from-workspace packaging script must not be restored."
|
||||
exit 1
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text.
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
echo "Runtime SSOT guard passed; core consumes the standalone runtime package."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
all-required:
|
||||
# Aggregator sentinel — RFC internal#219 §2 (Phase 4 — closes internal#286).
|
||||
#
|
||||
# Emits `CI / all-required (<event>)` where <event> is the workflow trigger
|
||||
# (e.g. `CI / all-required (pull_request)`, `CI / all-required (push)`).
|
||||
# Branch protection requires the event-suffixed name —
|
||||
# requiring `CI / all-required` (bare, no suffix) silently blocks all merges
|
||||
# because Gitea treats absent status contexts as pending (not skipped), and
|
||||
# no workflow emits the bare name. BP requires
|
||||
# `CI / all-required (pull_request)` per issue #1473.
|
||||
# Single stable required-status name that branch protection points at;
|
||||
# CI churns underneath in `needs:` without any protection edits. Mirrors
|
||||
# the molecule-controlplane Phase 2a impl shipped in CP PR#112 and
|
||||
# referenced by `internal#286` ("Phase 4 is a single small PR... mirrors
|
||||
# CP's existing one").
|
||||
#
|
||||
# Closes the failure mode where status_check_contexts on molecule-core/main
|
||||
# only listed `Secret scan` + `sop-tier-check` (the 2 meta-gates), so real
|
||||
@@ -487,91 +509,72 @@ jobs:
|
||||
# red silently merged through. See internal#286 for the three concrete
|
||||
# tonight-of-2026-05-11 incidents that prompted the emergency bump.
|
||||
#
|
||||
# ── 2026-06-01 CI-scheduler-overload fix (fix/ci-scheduler-fanout) ──
|
||||
# PREVIOUS shape: a poll-gate that ran detect-changes then LOOPED on
|
||||
# `GET /commits/{sha}/statuses` every 15s for up to 40 min, occupying a
|
||||
# `ci-meta` executor slot the entire time it waited for upstream jobs.
|
||||
# With only 2 ci-meta runners, that poll-loop squatted half the lane on
|
||||
# every PR — a confirmed throughput sink in the live RCA (two concurrent
|
||||
# `JOB-all-required` containers observed pinning the lane). The polling
|
||||
# design existed only to dodge the Gitea `needs:` + `if: always()` bug,
|
||||
# where an always()-guarded sentinel could be marked skipped before
|
||||
# upstream jobs settled (leaving BP pending forever).
|
||||
# Three properties of this job each close a failure mode:
|
||||
#
|
||||
# NEW shape: a plain `needs:` aggregator with NO polling loop. This is
|
||||
# safe here — and was NOT safe at the time the poller was written —
|
||||
# because every aggregated CI job now gates its real work PER-STEP
|
||||
# (`if: needs.changes.outputs.* != 'true'`) rather than at the JOB level.
|
||||
# A per-step-gated job always reaches a terminal SUCCESS (it no-ops its
|
||||
# expensive steps but the job itself still completes), so it is never
|
||||
# `skipped`. Plain `needs:` (WITHOUT `if: always()`) works correctly on
|
||||
# Gitea 1.22.6 / act_runner v0.6.1 — only `needs:` + `if: always()` is
|
||||
# broken (feedback_gitea_needs_works_only_ifalways_broken). We therefore
|
||||
# use plain `needs:` + an explicit per-need result check (NOT
|
||||
# `if: always()`); if any need fails/errors, Gitea never starts this job
|
||||
# and BP sees `CI / all-required` go red via the failed dependency
|
||||
# propagation — exactly the gate we want, with zero runner-squat.
|
||||
# 1. `if: always()` — runs even when an upstream fails. Without it the
|
||||
# sentinel is `skipped` and protection treats that as missing → merge
|
||||
# ungated.
|
||||
#
|
||||
# The `needs:` list MUST stay in lockstep with ci-required-drift.py's
|
||||
# F1 check (`ci_job_names()` = every job MINUS the sentinel MINUS jobs
|
||||
# whose `if:` gates on github.event_name/github.ref). canvas-deploy-
|
||||
# reminder is event-gated (`if: github.ref == refs/heads/{main,staging}`)
|
||||
# so it is intentionally EXCLUDED — it skips on PRs and a `needs:` on a
|
||||
# skipped job would never let the sentinel run. If a new always-running
|
||||
# CI job is added, add it here too or ci-required-drift F1 will flag it.
|
||||
# 2. Assertion is `result == "success"` per dep, NOT `!= "failure"`.
|
||||
# A `skipped` upstream (job gated by `if:` evaluating false, matrix
|
||||
# entry that couldn't run) must NOT silently pass through.
|
||||
# `skipped`-as-green is exactly the failure mode this gate closes.
|
||||
#
|
||||
# Stays on the dedicated `ci-meta` lane (no docker work, so the
|
||||
# docker-host-pin lint does not apply), but now the job is sub-second:
|
||||
# it only inspects already-settled `needs.*.result` values, so it frees
|
||||
# the slot immediately instead of holding it for the whole CI duration.
|
||||
# 3. `needs:` is the canonical list of "what counts as required."
|
||||
# status_check_contexts will reference only `ci/all-required` (Step 5
|
||||
# follow-up — branch-protection PATCH is Owners-tier per
|
||||
# `feedback_never_admin_merge_bypass`, separate PR); a new job is
|
||||
# added simply by listing it in `needs:` here.
|
||||
# `.gitea/workflows/ci-required-drift.yml` files a [ci-drift] issue
|
||||
# hourly if this list diverges from status_check_contexts or from
|
||||
# audit-force-merge.yml's REQUIRED_CHECKS env (RFC §4 + §6).
|
||||
#
|
||||
# Excluded from `needs:`: `canvas-deploy-reminder` — gated by
|
||||
# `if: ... github.event_name == 'push' && github.ref == 'refs/heads/main'`,
|
||||
# so on PR events it's legitimately `skipped`. The drift detector
|
||||
# explicitly excludes `github.event_name`-gated jobs from F1 (see
|
||||
# `.gitea/scripts/ci-required-drift.py::ci_job_names`).
|
||||
#
|
||||
# Phase 3 (RFC #219 §1) safety: continue-on-error here so the sentinel
|
||||
# does not hard-fail and block PRs while the underlying build jobs are
|
||||
# still in Phase 3 (continue-on-error: true suppresses their status to null).
|
||||
# When Phase 3 ends (defects fixed, continue-on-error flipped off on build
|
||||
# jobs), remove continue-on-error here so the sentinel again hard-fails.
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 1
|
||||
needs:
|
||||
- changes
|
||||
- platform-build
|
||||
- canvas-build
|
||||
- shellcheck
|
||||
- python-lint
|
||||
continue-on-error: false
|
||||
runs-on: ci-meta
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
steps:
|
||||
- name: Verify all aggregated CI jobs succeeded
|
||||
# NO polling, NO API call, NO checkout. Because this job lists the
|
||||
# aggregated jobs under `needs:` (without `if: always()`), Gitea only
|
||||
# starts it once every need has reached SUCCESS — a failed/errored
|
||||
# need short-circuits the job and propagates red to the
|
||||
# `CI / all-required` context. This explicit check is a
|
||||
# belt-and-suspenders assertion + a readable run summary; the real
|
||||
# gating is the `needs:` edge itself.
|
||||
env:
|
||||
CHANGES_RESULT: ${{ needs.changes.result }}
|
||||
PLATFORM_RESULT: ${{ needs.platform-build.result }}
|
||||
CANVAS_RESULT: ${{ needs.canvas-build.result }}
|
||||
SHELLCHECK_RESULT: ${{ needs.shellcheck.result }}
|
||||
PYTHON_LINT_RESULT: ${{ needs.python-lint.result }}
|
||||
- name: Assert every required dependency succeeded
|
||||
run: |
|
||||
set -euo pipefail
|
||||
fail=0
|
||||
check() {
|
||||
name="$1"; result="$2"
|
||||
printf 'CI / %s = %s\n' "$name" "$result"
|
||||
# `success` is the only green terminal state we accept. A plain
|
||||
# `needs:` job is only started when all needs succeed, so reaching
|
||||
# this step already implies success — but assert explicitly so a
|
||||
# future `if: always()` reintroduction (which WOULD let non-success
|
||||
# through) fails loudly instead of silently passing the gate.
|
||||
if [ "$result" != "success" ]; then
|
||||
echo "::error::aggregated CI job '${name}' did not succeed (result=${result})"
|
||||
fail=1
|
||||
fi
|
||||
}
|
||||
check "Detect changes" "$CHANGES_RESULT"
|
||||
check "Platform (Go)" "$PLATFORM_RESULT"
|
||||
check "Canvas (Next.js)" "$CANVAS_RESULT"
|
||||
check "Shellcheck (E2E scripts)" "$SHELLCHECK_RESULT"
|
||||
check "Python Lint & Test" "$PYTHON_LINT_RESULT"
|
||||
if [ "$fail" -ne 0 ]; then
|
||||
echo "::error::all-required: one or more aggregated CI jobs did not succeed"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: all aggregated CI jobs succeeded — CI / all-required green."
|
||||
# `needs.*.result` is one of: success | failure | cancelled | skipped | null.
|
||||
# We assert success per dep (not != failure) — see RFC §2 reasoning above.
|
||||
# Null results are skipped: they come from Phase 3 (continue-on-error: true
|
||||
# suppresses status) or from jobs still in-flight. The sentinel succeeds
|
||||
# rather than blocking PRs on Phase 3 noise.
|
||||
results='${{ toJSON(needs) }}'
|
||||
echo "$results"
|
||||
echo "$results" | python3 -c '
|
||||
import json, sys
|
||||
ns = json.load(sys.stdin)
|
||||
# Exclude null (Phase 3 suppressed / in-flight) from the bad list.
|
||||
bad = [(k, v.get("result")) for k, v in ns.items()
|
||||
if v.get("result") not in ("success", None)]
|
||||
if bad:
|
||||
print(f"FAIL: jobs not green:", file=sys.stderr)
|
||||
for k, r in bad:
|
||||
print(f" - {k}: {r}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
pending = [(k, v.get("result")) for k, v in ns.items() if v.get("result") is None]
|
||||
if pending:
|
||||
print(f"WARN: {len(pending)} job(s) still in-flight (result=null): " +
|
||||
", ".join(k for k, _ in pending), file=sys.stderr)
|
||||
print(f"OK: all {len(ns)} required jobs succeeded (or Phase-3 suppressed)")
|
||||
'
|
||||
|
||||
@@ -43,18 +43,6 @@ name: Continuous synthetic E2E (staging)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 minutes, on :02 and :32. This keeps a recurring SaaS
|
||||
# behavior probe while cutting runner occupancy from this workflow by
|
||||
# roughly two thirds; fast liveness belongs in the lighter smoke/heartbeat
|
||||
# probes, not in a full tenant/workspace synth every 10 minutes.
|
||||
#
|
||||
# Previous cadence was every 10 minutes (:02 :12 :22 :32 :42 :52).
|
||||
# The current operator-host runner pool is the bottleneck, so full
|
||||
# synth E2E is deliberately lower-cadence until it moves to a dedicated
|
||||
# runner host or warm-runtime pool.
|
||||
#
|
||||
# Historical notes from the 10-minute shape:
|
||||
#
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
@@ -78,7 +66,7 @@ on:
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,32 * * * *'
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
@@ -102,7 +90,6 @@ jobs:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
@@ -118,7 +105,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2 via the template's third-party-
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
@@ -131,9 +118,9 @@ jobs:
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2' }}
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
@@ -145,11 +132,6 @@ jobs:
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
@@ -166,10 +148,6 @@ jobs:
|
||||
# canary path. The script picks the right blob shape based on
|
||||
# which key is non-empty.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
# google-adk canary path — AI-Studio key (config model
|
||||
# google_genai:gemini-2.5-pro). PROD disallows API keys (Vertex+ADC);
|
||||
# the keyed path is CI-only. Dispatch with E2E_RUNTIME=google-adk.
|
||||
E2E_GOOGLE_API_KEY: ${{ secrets.MOLECULE_STAGING_GOOGLE_API_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -194,12 +172,6 @@ jobs:
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var secret missing — EC2 leak verification cannot run"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
@@ -221,10 +193,6 @@ jobs:
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
google-adk)
|
||||
required_secret_name="MOLECULE_STAGING_GOOGLE_API_KEY"
|
||||
required_secret_value="${E2E_GOOGLE_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
|
||||
@@ -69,13 +69,6 @@ name: E2E API Smoke Test
|
||||
# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
|
||||
# they DO come up. Timeouts are not the bottleneck; not bumped.
|
||||
#
|
||||
# Item #1046 (fixed 2026-05-14): Stale platform-server from cancelled runs
|
||||
# lingers on :8080 after "Stop platform" step is skipped (workflow cancelled
|
||||
# before reaching line 335). Added a pre-start "Kill stale platform-server"
|
||||
# step (line 286) that scans /proc for zombie platform-server processes
|
||||
# and kills them before the port probe or bind. Makes the ephemeral port
|
||||
# probe + start sequence deterministic.
|
||||
#
|
||||
# Item explicitly NOT fixed here: failing test `Status back online`
|
||||
# fails because the platform's langgraph workspace template image
|
||||
# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
|
||||
@@ -108,22 +101,8 @@ env:
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
# mc#1529 follow-on: pin to `docker-host` so the e2e-api lane lands
|
||||
# on Linux operator-host runners (molecule-runner-*) that carry the
|
||||
# `molecule-core-net` bridge network + a working `aws ecr get-login-
|
||||
# password | docker login` path. The bare `ubuntu-latest` label is
|
||||
# also accepted by hongming-pc-runner-* (Windows act_runner v1.0.3),
|
||||
# where the docker.sock-bound steps below fail non-deterministically
|
||||
# (e.g. `docker run -d --name pg-e2e-api-...` with port-bind +
|
||||
# `docker exec ... pg_isready` cannot work against a Windows daemon).
|
||||
# detect-changes itself doesn't bind docker.sock, but pinning here too
|
||||
# keeps both jobs on the same lane so we don't re-roll the dice on
|
||||
# workspace-volume cross-host surprises and the routing rule is
|
||||
# discoverable in one place. Mirror of mc#1543 (handlers-postgres-
|
||||
# integration). See internal#512 for the class defect.
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
api: ${{ steps.decide.outputs.api }}
|
||||
@@ -132,13 +111,31 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
# Inline replacement for dorny/paths-filter — same pattern PR#372's
|
||||
# ci.yml port used. Diffs against the PR base or push BEFORE SHA,
|
||||
# then matches against the api-relevant path set.
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile e2e-api \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "${{ github.event.pull_request.base.sha }}" \
|
||||
--base-ref "${{ github.event.pull_request.base.ref }}" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-${{ github.event.before }}}"
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/|tests/e2e/|\.gitea/workflows/e2e-api\.yml$)'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
@@ -155,12 +152,8 @@ jobs:
|
||||
e2e-api:
|
||||
needs: detect-changes
|
||||
name: E2E API Smoke Test
|
||||
# mc#1529 follow-on: must run on operator-host Linux runners (where
|
||||
# docker.sock + `molecule-core-net` + `aws ecr ...` work). See
|
||||
# detect-changes for the full rationale.
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
@@ -171,6 +164,7 @@ jobs:
|
||||
# we let Docker assign an ephemeral host port.
|
||||
PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
PORT: "8080"
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.api != 'true'
|
||||
@@ -274,49 +268,6 @@ jobs:
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Pick platform port
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
PLATFORM_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "BASE=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Platform host port: ${PLATFORM_PORT}"
|
||||
- name: Kill stale platform-server before start (issue #1046)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Concurrent runs on the same host-network act_runner can leave a
|
||||
# zombie platform-server from a cancelled/timeout run. Cancelled
|
||||
# runs never reach the "Stop platform" step (line 335), so the
|
||||
# old process lingers. Kill it before the ephemeral port probe
|
||||
# or start so the port is definitively free.
|
||||
#
|
||||
# /proc scan — works on any Linux without pkill/lsof/ss.
|
||||
# comm field is truncated to 15 chars: "platform-serve" matches
|
||||
# "platform-server". Verify with cmdline to avoid false positives.
|
||||
killed=0
|
||||
for pid in $(grep -l "platform-serve" /proc/[0-9]*/comm 2>/dev/null); do
|
||||
kpid="${pid%/comm}"
|
||||
kpid="${kpid##*/}"
|
||||
cmdline=$(cat "/proc/${kpid}/cmdline" 2>/dev/null | tr '\0' ' ')
|
||||
if echo "$cmdline" | grep -q "platform-server"; then
|
||||
echo "Killing stale platform-server pid ${kpid}: ${cmdline}"
|
||||
kill "$kpid" 2>/dev/null || true
|
||||
killed=$((killed + 1))
|
||||
fi
|
||||
done
|
||||
if [ "$killed" -gt 0 ]; then
|
||||
sleep 2
|
||||
echo "Killed $killed stale process(es); port(s) released."
|
||||
else
|
||||
echo "No stale platform-server found."
|
||||
fi
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
@@ -329,7 +280,7 @@ jobs:
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "$BASE/health" > /dev/null; then
|
||||
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
@@ -348,9 +299,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
- name: Run today's-PR-coverage E2E (mc#1525/1535/1536/1539/1542 fix-specific assertions)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_today_pr_coverage_e2e.sh
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
@@ -360,12 +308,6 @@ jobs:
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Install standalone runtime parser from Gitea registry
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
python3 -m pip install --no-deps \
|
||||
--index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ \
|
||||
molecule-ai-workspace-runtime
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
name: E2E Chat
|
||||
|
||||
# Comprehensive Playwright E2E for the unified chat stack (desktop
|
||||
# ChatTab + mobile MobileChat). Heavy browser execution is intentionally
|
||||
# outside the normal required PR path: PRs run it only after entering the
|
||||
# `merge-queue`, while push/main, nightly, and manual dispatch preserve
|
||||
# coverage without making every PR pay the full runtime/browser cost.
|
||||
#
|
||||
# Architecture:
|
||||
# 1. Ephemeral Postgres + Redis (docker, unique container names)
|
||||
# 2. workspace-server built from source, started with
|
||||
# MOLECULE_ENV=development (fail-open auth)
|
||||
# 3. canvas dev server (npm run dev) on :3000
|
||||
# 4. Playwright tests create workspaces via API, point them at an
|
||||
# in-process echo runtime, and exercise the full send/receive
|
||||
# round-trip through the browser.
|
||||
#
|
||||
# Parallel-safety: same pattern as e2e-api.yml — per-run container names
|
||||
# and ephemeral host ports so concurrent jobs on the host-network runner
|
||||
# don't collide.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
schedule:
|
||||
# Nightly at 09:00 UTC. Keeps coverage for the currently non-required
|
||||
# heavy browser lane without spending runner time on every PR.
|
||||
- cron: '0 9 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: e2e-chat-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: helper job; real gate is E2E Chat / E2E Chat (pull_request)
|
||||
detect-changes:
|
||||
# mc#1529 follow-on: pin to `docker-host` (Linux operator-host
|
||||
# runners). The bare `ubuntu-latest` label is also advertised by
|
||||
# hongming-pc-runner-* (Windows act_runner v1.0.3) where the
|
||||
# docker.sock-bound steps below fail. Mirror of mc#1543
|
||||
# (handlers-postgres-integration). See internal#512 for the class
|
||||
# defect.
|
||||
runs-on: docker-host
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
chat: ${{ steps.decide.outputs.chat }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if ! echo "$CHANGED" | grep -qE '^(canvas/|workspace-server/|\.gitea/workflows/e2e-chat\.yml$)'; then
|
||||
echo "chat=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ github.event_name }}" != "pull_request" ]; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
labels=$(curl -fsS -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
|
||||
| python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
|
||||
rm -f "$authfile"
|
||||
if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "PR is not in merge-queue; skipping heavy E2E Chat for normal PR path."
|
||||
echo "chat=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# bp-required: pending #1142 — new E2E check; add to branch protection after 3 green runs.
|
||||
e2e-chat:
|
||||
needs: detect-changes
|
||||
name: E2E Chat
|
||||
# mc#1529 follow-on: docker run/exec for postgres + redis containers.
|
||||
# Must land on operator-host Linux (docker-host).
|
||||
runs-on: docker-host
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
PG_CONTAINER: pg-e2e-chat-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-chat-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.chat != 'true'
|
||||
run: |
|
||||
echo "No canvas / workspace-server / workflow changes — E2E Chat gate satisfied without running tests."
|
||||
echo "::notice::E2E Chat no-op pass (paths filter excluded this commit)."
|
||||
|
||||
- if: needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- if: needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
|
||||
- if: needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: canvas/package-lock.json
|
||||
|
||||
- name: Start Postgres (docker)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
exit 1
|
||||
fi
|
||||
echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "E2E_DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
|
||||
echo "Postgres ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"
|
||||
exit 1
|
||||
|
||||
- name: Start Redis (docker)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 15); do
|
||||
if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Redis ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"
|
||||
exit 1
|
||||
|
||||
- name: Build platform
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
|
||||
- name: Pick platform port
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
PLATFORM_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "PLATFORM_PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "E2E_PLATFORM_URL=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Platform host port: ${PLATFORM_PORT}"
|
||||
|
||||
- name: Pick canvas port
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
CANVAS_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "CANVAS_PORT=${CANVAS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Canvas host port: ${CANVAS_PORT}"
|
||||
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
export MOLECULE_ENV=development
|
||||
export DATABASE_URL="${DATABASE_URL}"
|
||||
export REDIS_URL="${REDIS_URL}"
|
||||
export PORT="${PLATFORM_PORT}"
|
||||
export CORS_ORIGINS="http://localhost:3000,http://localhost:3001,http://localhost:${CANVAS_PORT},http://127.0.0.1:${CANVAS_PORT}"
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
|
||||
- name: Wait for /health
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "http://127.0.0.1:${PLATFORM_PORT}/health" > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
|
||||
- name: Install canvas dependencies
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: |
|
||||
PREBAKED_PLAYWRIGHT=/ms-playwright
|
||||
if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
|
||||
echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
|
||||
echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
|
||||
exit 0
|
||||
fi
|
||||
npx playwright install --with-deps chromium
|
||||
|
||||
- name: Start canvas dev server (background)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: |
|
||||
export NEXT_PUBLIC_PLATFORM_URL="http://127.0.0.1:${PLATFORM_PORT}"
|
||||
export NEXT_PUBLIC_WS_URL="ws://127.0.0.1:${PLATFORM_PORT}/ws"
|
||||
npx next dev --turbopack -p "${CANVAS_PORT}" > canvas.log 2>&1 &
|
||||
echo $! > canvas.pid
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "http://localhost:${CANVAS_PORT}" > /dev/null 2>&1; then
|
||||
echo "Canvas up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Canvas did not start in 30s"
|
||||
cat canvas.log || true
|
||||
exit 1
|
||||
|
||||
- name: Run Playwright E2E tests
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: |
|
||||
export E2E_PLATFORM_URL="http://127.0.0.1:${PLATFORM_PORT}"
|
||||
export E2E_DATABASE_URL="${DATABASE_URL}"
|
||||
export PLAYWRIGHT_BASE_URL="http://localhost:${CANVAS_PORT}"
|
||||
npx playwright test e2e/chat-desktop.spec.ts e2e/chat-mobile.spec.ts
|
||||
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: cat workspace-server/platform.log || true
|
||||
|
||||
- name: Dump canvas log on failure
|
||||
if: failure() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: cat canvas/canvas.log || true
|
||||
|
||||
- name: Upload Playwright report
|
||||
if: failure() && needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/upload-artifact@v3.2.2
|
||||
with:
|
||||
name: playwright-report-chat
|
||||
path: canvas/playwright-report/
|
||||
|
||||
- name: Stop canvas
|
||||
if: always() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
if [ -f canvas/canvas.pid ]; then
|
||||
kill "$(cat canvas/canvas.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Stop platform
|
||||
if: always() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Stop service containers
|
||||
if: always() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
@@ -1,242 +0,0 @@
|
||||
name: E2E Legacy Advisory
|
||||
|
||||
# Advisory lane for older/manual E2E scripts that are too broad or
|
||||
# environment-dependent for required PR CI. This intentionally does not run on
|
||||
# pull_request or push so it cannot block merges/deploys; scheduled/manual reds
|
||||
# still surface drift in scripts that would otherwise only be shellchecked.
|
||||
#
|
||||
# Gitea 1.22.6 rejects workflow_dispatch.inputs, so keep dispatch input-free.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Stagger after the staging smoke/canvas morning lanes.
|
||||
- cron: '15 9 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: e2e-legacy-advisory
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
legacy-local-platform:
|
||||
name: Legacy local-platform E2E
|
||||
runs-on: docker-host
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
PG_CONTAINER: pg-e2e-legacy-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-legacy-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
MOLECULE_ENV: development
|
||||
BIND_ADDR: 127.0.0.1
|
||||
MOLECULE_IN_DOCKER: "false"
|
||||
A2A_TIMEOUT: "30"
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
|
||||
- name: Prepare local platform dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker pull postgres:16 >/dev/null
|
||||
docker pull redis:7 >/dev/null
|
||||
docker pull alpine:latest >/dev/null
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
|
||||
- name: Start Postgres
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker port "$PG_CONTAINER" 5432/tcp || true
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1 && exit 0
|
||||
sleep 1
|
||||
done
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
|
||||
- name: Start Redis
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker port "$REDIS_CONTAINER" 6379/tcp || true
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 15); do
|
||||
docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG && exit 0
|
||||
sleep 1
|
||||
done
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
|
||||
- name: Pick platform port
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PLATFORM_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "BASE=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build platform
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
|
||||
- name: Populate template manifests for dev-mode E2E
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
bash scripts/clone-manifest.sh manifest.json workspace-configs-templates org-templates plugins
|
||||
else
|
||||
echo "::warning::jq unavailable; dev-mode template assertion may fail if templates are absent"
|
||||
fi
|
||||
|
||||
- name: Start platform
|
||||
run: |
|
||||
set -euo pipefail
|
||||
./workspace-server/platform-server > workspace-server/platform.log 2>&1 &
|
||||
echo $! > workspace-server/platform.pid
|
||||
for i in $(seq 1 30); do
|
||||
curl -sf "$BASE/health" >/dev/null && exit 0
|
||||
sleep 1
|
||||
done
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
|
||||
- name: Run comprehensive E2E
|
||||
run: bash tests/e2e/test_comprehensive_e2e.sh
|
||||
|
||||
- name: Run workspace abilities E2E
|
||||
run: bash tests/e2e/test_workspace_abilities_e2e.sh
|
||||
|
||||
- name: Run dev-mode E2E
|
||||
run: bash tests/e2e/test_dev_mode.sh
|
||||
|
||||
- name: Start stub A2A agents
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cat > /tmp/molecule-stub-a2a.py <<'PY'
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("content-length", "0"))
|
||||
raw = self.rfile.read(length) if length else b"{}"
|
||||
try:
|
||||
req = json.loads(raw)
|
||||
except Exception:
|
||||
req = {}
|
||||
method = req.get("method")
|
||||
if method not in ("message/send", None):
|
||||
body = {"jsonrpc": "2.0", "id": req.get("id"), "error": {"code": -32601, "message": "method not found"}}
|
||||
else:
|
||||
body = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": req.get("id", "stub"),
|
||||
"result": {
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "type": "text", "text": "stub agent response"}],
|
||||
},
|
||||
}
|
||||
data = json.dumps(body, separators=(",", ":")).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("content-type", "application/json")
|
||||
self.send_header("content-length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
def log_message(self, *_):
|
||||
return
|
||||
|
||||
HTTPServer(("127.0.0.1", 18080), Handler).serve_forever()
|
||||
PY
|
||||
python3 /tmp/molecule-stub-a2a.py > /tmp/molecule-stub-a2a.log 2>&1 &
|
||||
echo $! > /tmp/molecule-stub-a2a.pid
|
||||
|
||||
- name: Seed external agents for legacy A2A/activity scripts
|
||||
run: |
|
||||
set -euo pipefail
|
||||
create_agent() {
|
||||
local name="$1" role="$2"
|
||||
curl -sS -X POST "$BASE/workspaces" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"name\":\"${name}\",\"role\":\"${role}\",\"tier\":1,\"runtime\":\"external\",\"external\":true,\"url\":\"http://127.0.0.1:18080\"}" \
|
||||
| python3 -c "import json,sys; print(json.load(sys.stdin)['id'])"
|
||||
}
|
||||
ECHO_ID=$(create_agent "Echo Agent" "Echo")
|
||||
SEO_ID=$(create_agent "SEO Agent" "SEO")
|
||||
curl -sS -X POST "$BASE/registry/register" -H "Content-Type: application/json" \
|
||||
-d "{\"id\":\"$ECHO_ID\",\"url\":\"http://127.0.0.1:18080\",\"agent_card\":{\"name\":\"Echo Agent\",\"skills\":[{\"id\":\"echo\",\"name\":\"Echo\"}]}}" >/dev/null
|
||||
curl -sS -X POST "$BASE/registry/register" -H "Content-Type: application/json" \
|
||||
-d "{\"id\":\"$SEO_ID\",\"url\":\"http://127.0.0.1:18080\",\"agent_card\":{\"name\":\"SEO Agent\",\"skills\":[{\"id\":\"seo\",\"name\":\"SEO\"}]}}" >/dev/null
|
||||
|
||||
- name: Run activity E2E
|
||||
run: bash tests/e2e/test_activity_e2e.sh
|
||||
|
||||
- name: Run A2A E2E
|
||||
run: bash tests/e2e/test_a2a_e2e.sh
|
||||
|
||||
- name: Runtime-dependent legacy E2E preflight
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -f workspace-configs-templates/claude-code-default/.auth-token ] && docker image inspect workspace:latest >/dev/null 2>&1; then
|
||||
bash tests/e2e/test_claude_code_e2e.sh
|
||||
bash tests/e2e/test_chat_upload_e2e.sh
|
||||
else
|
||||
echo "::notice::Skipping test_claude_code_e2e.sh and test_chat_upload_e2e.sh: require workspace:latest plus workspace-configs-templates/claude-code-default/.auth-token"
|
||||
fi
|
||||
|
||||
- name: Dump platform log on failure
|
||||
if: failure()
|
||||
run: cat workspace-server/platform.log || true
|
||||
|
||||
- name: Stop platform and stub agents
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
if [ -f /tmp/molecule-stub-a2a.pid ]; then
|
||||
kill "$(cat /tmp/molecule-stub-a2a.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Stop service containers
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
@@ -1,403 +0,0 @@
|
||||
name: E2E Peer Visibility (literal MCP list_peers)
|
||||
|
||||
# WHY A DEDICATED WORKFLOW (not folded into e2e-staging-saas.yml)
|
||||
# --------------------------------------------------------------
|
||||
# This is the systemic fix for a real trust failure. Hermes and OpenClaw
|
||||
# were reported "fleet-verified / cascade-complete" because the *proxy*
|
||||
# signals were green (registry registration + heartbeat for Hermes; model
|
||||
# round-trip 200 for OpenClaw). A freshly-provisioned workspace asked on
|
||||
# canvas "can you see your peers" actually FAILS:
|
||||
# - Hermes: 401 on the molecule MCP `list_peers` call
|
||||
# - OpenClaw: native `sessions_list` fallback, sees no platform peers
|
||||
# Tasks #142/#159 were even marked "completed" under this proxy flaw.
|
||||
#
|
||||
# A dedicated workflow (vs extending e2e-staging-saas.yml) because:
|
||||
# - It must provision MULTIPLE distinct runtimes (hermes, openclaw,
|
||||
# claude-code) in ONE org and assert each sees the others. The
|
||||
# full-saas script is single-runtime-per-run (E2E_RUNTIME) and folding
|
||||
# a multi-runtime matrix into it would conflate concerns and bloat its
|
||||
# already-45-min run.
|
||||
# - It needs its own concurrency group so it doesn't fight full-saas /
|
||||
# canvas for the staging org-creation quota.
|
||||
# - It needs an independent, non-required status-context name so it can
|
||||
# be RED today (the in-flight Hermes-401 / OpenClaw-MCP-wiring fixes
|
||||
# have not landed) WITHOUT wedging unrelated merges — and flipped to
|
||||
# REQUIRED in one branch-protection edit once it goes green
|
||||
# (flip-to-required checklist: molecule-core#1296).
|
||||
#
|
||||
# THE ASSERTION IS NOT A PROXY. The driving script
|
||||
# tests/e2e/test_peer_visibility_mcp_staging.sh issues the byte-for-byte
|
||||
# JSON-RPC `tools/call name=list_peers` envelope to `POST
|
||||
# /workspaces/:id/mcp` using each workspace's OWN bearer token, through
|
||||
# the real WorkspaceAuth + MCPRateLimiter middleware chain — the exact
|
||||
# call mcp_molecule_list_peers makes from a canvas agent. It does NOT
|
||||
# read a registry row, /health, the heartbeat table, or
|
||||
# GET /registry/:id/peers.
|
||||
#
|
||||
# HONEST GATE — NO continue-on-error. Per feedback_fix_root_not_symptom a
|
||||
# fake-green mask would defeat the entire purpose. This workflow goes red
|
||||
# on today's broken behavior and green only when the root-cause fixes
|
||||
# actually land. It is intentionally NOT in branch_protections — see PR
|
||||
# body for the required-vs-not decision + flip tracking issue.
|
||||
#
|
||||
# Gitea 1.22.6 / act_runner notes honored:
|
||||
# - No cross-repo `uses:` (feedback_gitea_cross_repo_uses_blocked). The
|
||||
# actions/checkout SHA is the one e2e-staging-canvas.yml already uses
|
||||
# successfully (a mirrored SHA — see #1277/PR#1292 root-cause).
|
||||
# - 2026-05-21 retrigger: verify fresh platform-tenant image after the
|
||||
# publish Buildx DOCKER_CONFIG fix restored staging-latest image updates.
|
||||
# - Per-SHA concurrency, not global (feedback_concurrency_group_per_sha).
|
||||
# - Workflow-level GITHUB_SERVER_URL pinned
|
||||
# (feedback_act_runner_github_server_url).
|
||||
# - pr-validate posts a status under the same check name so a
|
||||
# workflow-only PR is not silently statusless and the context is
|
||||
# flip-to-required-ready (mirrors e2e-staging-saas.yml's proven shape;
|
||||
# real EC2-provisioning E2E is push/dispatch/cron only — it is 30+ min
|
||||
# and cannot run per-PR-update).
|
||||
#
|
||||
# LOCAL BACKEND (added 2026-05-15 — feedback_local_must_mimic_production,
|
||||
# feedback_mandatory_local_e2e_before_ship, feedback_local_test_before_
|
||||
# staging_e2e)
|
||||
# --------------------------------------------------------------------
|
||||
# The standing rule is that the local prod-mimic stack runs a MANDATORY
|
||||
# local-Postgres E2E BEFORE staging E2E. A staging-only peer-visibility
|
||||
# gate caught regressions late + expensively (cold EC2). The
|
||||
# `peer-visibility-local` job below runs the SAME byte-identical
|
||||
# assertion (tests/e2e/lib/peer_visibility_assert.sh) against the local
|
||||
# docker-compose stack — built + booted exactly like e2e-api.yml's
|
||||
# proven E2E API Smoke Test job (ephemeral pg/redis ports, go build,
|
||||
# background platform-server). It runs on PR + push (local boot is
|
||||
# minutes, not the 30+ min cold-EC2 path), so peer-visibility is part of
|
||||
# the local gate that fires before the staging E2E.
|
||||
#
|
||||
# It is its OWN non-required status context `E2E Peer Visibility (local)`.
|
||||
# The local backend uses external-mode workspaces by default so it tests
|
||||
# the literal platform MCP list_peers path without depending on local
|
||||
# template container boot/heartbeat. Container-mode runtime boot remains
|
||||
# available via PV_LOCAL_PROVISION_MODE=container for targeted debugging.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/mcp.go'
|
||||
- 'workspace-server/internal/handlers/mcp_tools.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_local.sh'
|
||||
- 'tests/e2e/lib/peer_visibility_assert.sh'
|
||||
- '.gitea/workflows/e2e-peer-visibility.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/mcp.go'
|
||||
- 'workspace-server/internal/handlers/mcp_tools.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_local.sh'
|
||||
- 'tests/e2e/lib/peer_visibility_assert.sh'
|
||||
- '.gitea/workflows/e2e-peer-visibility.yml'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 07:30 UTC daily — catches AMI / template-hermes / template-openclaw
|
||||
# drift even on quiet days. Offset 30m from e2e-staging-saas (07:00)
|
||||
# so the two don't collide on the staging org-creation quota.
|
||||
- cron: '30 7 * * *'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA (feedback_concurrency_group_per_sha). A single global group
|
||||
# would let a queued staging/main push behind a PR run get cancelled,
|
||||
# leaving any gate that reads "completed run at SHA" stuck.
|
||||
group: e2e-peer-visibility-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# PR path: post a real status under the required-ready check name so a
|
||||
# workflow-only PR is never silently statusless. The actual EC2 E2E is
|
||||
# push/dispatch/cron only (30+ min). This is NOT a fake-green mask of
|
||||
# the real assertion — it validates the driving script's bash syntax
|
||||
# and inline-python so a broken test script fails at PR time.
|
||||
pr-validate:
|
||||
name: E2E Peer Visibility
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Validate driving scripts + shared assertion lib
|
||||
run: |
|
||||
bash -n tests/e2e/lib/peer_visibility_assert.sh
|
||||
echo "lib/peer_visibility_assert.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_mcp_staging.sh
|
||||
echo "test_peer_visibility_mcp_staging.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_token_mint_staging.sh
|
||||
echo "test_peer_visibility_token_mint_staging.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_mcp_local.sh
|
||||
echo "test_peer_visibility_mcp_local.sh — bash syntax OK"
|
||||
legacy_token_suffix="test""-token"
|
||||
if rg -n "$legacy_token_suffix" tests/e2e/test_*staging*.sh; then
|
||||
echo "::error::staging E2E must use production-safe admin token minting"
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging fresh-provision MCP list_peers E2E runs on push to"
|
||||
echo "main / workflow_dispatch / daily cron (30+ min EC2 boot)."
|
||||
echo "The LOCAL backend runs in the peer-visibility-local job"
|
||||
echo "below on this same PR (local docker-compose stack)."
|
||||
|
||||
# LOCAL gate: same byte-identical assertion against the local prod-mimic
|
||||
# docker-compose stack — the MANDATORY local-E2E that must run BEFORE
|
||||
# the staging E2E (feedback_mandatory_local_e2e_before_ship,
|
||||
# feedback_local_test_before_staging_e2e). Bootstrap mirrors
|
||||
# e2e-api.yml's proven E2E API Smoke Test job (per-run container names +
|
||||
# ephemeral host ports so concurrent host-network act_runner runs don't
|
||||
# collide; go build; background platform-server). Its OWN non-required
|
||||
# status context `E2E Peer Visibility (local)` — non-required-by-design
|
||||
# exactly like the staging job (flip-to-required tracked at
|
||||
# molecule-core#1296). HONEST gate, NO continue-on-error mask
|
||||
# (feedback_fix_root_not_symptom). Runs on PR +
|
||||
# push (local boot is minutes, not the 30+ min cold-EC2 path).
|
||||
# bp-required: pending #1296
|
||||
peer-visibility-local:
|
||||
name: E2E Peer Visibility (local)
|
||||
runs-on: docker-host
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# Per-run names + ephemeral ports — same collision-avoidance as
|
||||
# e2e-api.yml (host-network act_runner; feedback_act_runner_*).
|
||||
PG_CONTAINER: pg-e2e-pv-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-pv-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# LLM keys so hermes/openclaw can actually boot. The local script
|
||||
# SKIPs (not fails) any runtime whose key is absent, so a partially
|
||||
# keyed CI env still exercises whatever it can.
|
||||
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.E2E_CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
PV_RUNTIMES: "hermes openclaw claude-code"
|
||||
PV_LOCAL_PROVISION_MODE: external
|
||||
ADMIN_TOKEN: local-e2e-admin-token
|
||||
MOLECULE_ADMIN_TOKEN: local-e2e-admin-token
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
- name: Pre-pull alpine + ensure provisioner network
|
||||
run: |
|
||||
docker pull alpine:latest >/dev/null
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
echo "alpine:latest pre-pulled; molecule-core-net ensured."
|
||||
- name: Start Postgres (docker, ephemeral port)
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
[ -n "$PG_PORT" ] || PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker logs "$PG_CONTAINER" || true; exit 1
|
||||
fi
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1 && { echo "Postgres ready after ${i}s"; exit 0; }
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"; docker logs "$PG_CONTAINER" || true; exit 1
|
||||
- name: Start Redis (docker, ephemeral port)
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
[ -n "$REDIS_PORT" ] || REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker logs "$REDIS_CONTAINER" || true; exit 1
|
||||
fi
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 15); do
|
||||
docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG && { echo "Redis ready after ${i}s"; exit 0; }
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"; docker logs "$REDIS_CONTAINER" || true; exit 1
|
||||
- name: Build platform
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Pick platform port
|
||||
run: |
|
||||
PLATFORM_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "BASE=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Platform host port: ${PLATFORM_PORT}"
|
||||
- name: Kill stale platform-server before start
|
||||
run: |
|
||||
killed=0
|
||||
for pid in $(grep -l "platform-serve" /proc/[0-9]*/comm 2>/dev/null); do
|
||||
kpid="${pid%/comm}"; kpid="${kpid##*/}"
|
||||
cmdline=$(cat "/proc/${kpid}/cmdline" 2>/dev/null | tr '\0' ' ')
|
||||
if echo "$cmdline" | grep -q "platform-server"; then
|
||||
echo "Killing stale platform-server pid ${kpid}"
|
||||
kill "$kpid" 2>/dev/null || true; killed=$((killed + 1))
|
||||
fi
|
||||
done
|
||||
[ "$killed" -gt 0 ] && sleep 2 || true
|
||||
echo "stale-kill done ($killed killed)"
|
||||
- name: Start platform (background)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
- name: Wait for /health
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
curl -sf "$BASE/health" > /dev/null && { echo "Platform up after ${i}s"; exit 0; }
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true; exit 1
|
||||
- name: Run LOCAL fresh-provision peer-visibility E2E (literal MCP list_peers)
|
||||
# HONEST gate — NO continue-on-error. The local backend uses
|
||||
# external-mode workspaces so this context tests the literal MCP
|
||||
# peer-visibility path without coupling to template container boot.
|
||||
run: bash tests/e2e/test_peer_visibility_mcp_local.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure()
|
||||
run: cat workspace-server/platform.log || true
|
||||
- name: Stop platform
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Stop service containers
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
|
||||
# Real STAGING gate: provisions a throwaway org + sibling-per-runtime,
|
||||
# drives the LITERAL list_peers MCP call per runtime, asserts 200 +
|
||||
# expected peer set, then scoped teardown. push(main)/dispatch/cron only.
|
||||
peer-visibility:
|
||||
name: E2E Peer Visibility
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name != 'pull_request'
|
||||
timeout-minutes: 60
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# LLM provider key so each runtime can authenticate at boot.
|
||||
# Priority MiniMax → direct-Anthropic → OpenAI matches
|
||||
# test_staging_full_saas.sh's secrets-injection chain.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
PV_RUNTIMES: "hermes openclaw claude-code"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present"
|
||||
|
||||
- name: Verify an LLM key present
|
||||
run: |
|
||||
if [ -z "${E2E_MINIMAX_API_KEY:-}" ] && [ -z "${E2E_ANTHROPIC_API_KEY:-}" ] && [ -z "${E2E_OPENAI_API_KEY:-}" ]; then
|
||||
echo "::error::No LLM provider key set — workspaces fail at boot with 'No provider API key found'. Set MOLECULE_STAGING_MINIMAX_API_KEY (or ANTHROPIC / OPENAI)."
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (HTTP $code) — infra, not a workspace bug. Failing loud per feedback_fix_root_not_symptom."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy"
|
||||
|
||||
- name: Run fresh-provision peer-visibility E2E (literal MCP list_peers)
|
||||
run: bash tests/e2e/test_peer_visibility_mcp_staging.sh
|
||||
|
||||
# Belt-and-braces scoped teardown: the script installs an EXIT/INT/
|
||||
# TERM trap, but if the runner itself is cancelled the trap may not
|
||||
# fire. This always() step deletes ONLY the e2e-pv-<run_id> org this
|
||||
# run created — never a cluster-wide sweep
|
||||
# (feedback_never_run_cluster_cleanup_tests_on_live_platform). The
|
||||
# admin DELETE is idempotent so double-invoking is safe;
|
||||
# sweep-stale-e2e-orgs is the final net (slug starts with 'e2e-').
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
except Exception:
|
||||
print(''); sys.exit(0)
|
||||
# ONLY sweep slugs from THIS run. e2e-pv-<YYYYMMDD>-<run_id>-...
|
||||
# Sweep today AND yesterday's UTC date so a midnight-crossing run
|
||||
# still matches its own slug (same bug class as the saas/canvas
|
||||
# safety nets).
|
||||
today = datetime.date.today()
|
||||
yest = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yest.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-pv-{dt}-{run_id}-' for dt in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-pv-{dt}-' for dt in dates)
|
||||
orgs = d if isinstance(d, list) else d.get('orgs', [])
|
||||
cands = [o['slug'] for o in orgs
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('instance_status') not in ('purged',)]
|
||||
print('\n'.join(cands))
|
||||
" 2>/dev/null)
|
||||
for slug in $orgs; do
|
||||
echo "Safety-net teardown: $slug"
|
||||
set +e
|
||||
curl -sS -o /tmp/pv-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/pv-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/pv-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::pv teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within MAX_AGE_MINUTES. Body: $(head -c 300 /tmp/pv-cleanup.out 2>/dev/null)"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
@@ -16,9 +16,9 @@ name: E2E Staging Canvas (Playwright)
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main, PR touching canvas sources + this workflow only
|
||||
# after the PR enters `merge-queue`, manual dispatch, and scheduled cron to
|
||||
# catch browser/runtime drift even when canvas is quiet.
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
@@ -37,10 +37,9 @@ on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# Nightly at 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * *'
|
||||
workflow_dispatch:
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
@@ -71,7 +70,6 @@ jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
canvas: ${{ steps.decide.outputs.canvas }}
|
||||
@@ -80,13 +78,10 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
# Cron and manual triggers always run real work (no diff context).
|
||||
# Cron triggers always run real work (no diff context).
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
if [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
@@ -106,26 +101,9 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if ! echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ github.event_name }}" != "pull_request" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
labels=$(curl -fsS -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
|
||||
| python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
|
||||
rm -f "$authfile"
|
||||
if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
|
||||
if echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "PR is not in merge-queue; skipping heavy E2E Staging Canvas for normal PR path."
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
@@ -140,7 +118,6 @@ jobs:
|
||||
name: Canvas tabs E2E
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 40
|
||||
|
||||
@@ -189,15 +166,7 @@ jobs:
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
PREBAKED_PLAYWRIGHT=/ms-playwright
|
||||
if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
|
||||
echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
|
||||
echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
|
||||
exit 0
|
||||
fi
|
||||
npx playwright install --with-deps chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
|
||||
@@ -84,7 +84,6 @@ jobs:
|
||||
name: E2E Staging External Runtime
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
|
||||
|
||||
@@ -49,9 +49,6 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/completion_assert.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
@@ -62,9 +59,6 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/completion_assert.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
@@ -94,31 +88,27 @@ jobs:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
|
||||
- name: YAML validation (best-effort)
|
||||
run: |
|
||||
echo "e2e-staging-saas.yml — PR validation: workflow YAML is valid."
|
||||
echo "E2E step runs only when provisioning-critical files change."
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
|
||||
# Actual E2E: runs on trunk pushes and PRs that touch provisioning-critical
|
||||
# paths. pr-validate remains as the lightweight workflow-shape check for PRs,
|
||||
# but it is not a substitute for live staging proof when this workflow or the
|
||||
# staging harness changes.
|
||||
# Actual E2E: runs on trunk pushes (main + staging). NOT the PR-fire-only
|
||||
# path — pr-validate above posts success for workflow-only PRs.
|
||||
e2e-staging-saas:
|
||||
name: E2E Staging SaaS
|
||||
runs-on: ubuntu-latest
|
||||
# Only runs on trunk pushes. PR paths get pr-validate instead.
|
||||
if: github.event.pull_request.base.ref == ''
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
@@ -133,11 +123,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -154,21 +139,16 @@ jobs:
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes or =codex via workflow_dispatch can still
|
||||
# E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
|
||||
# exercise the OpenAI path.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
# google-adk (operator-dispatched only) auths Gemini with an
|
||||
# AI-Studio key. Org policy disallows API keys in PROD (Vertex+ADC
|
||||
# there); CI uses the keyed AI-Studio path with config model
|
||||
# google_genai:gemini-2.5-pro. Vertex remains the supported prod path.
|
||||
E2E_GOOGLE_API_KEY: ${{ secrets.MOLECULE_STAGING_GOOGLE_API_KEY }}
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the model when running on the default claude-code path —
|
||||
# the per-runtime default ("sonnet") routes to direct Anthropic
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'codex' && 'openai/gpt-4o' || github.event.inputs.runtime == 'google-adk' && 'google_genai:gemini-2.5-pro' || 'MiniMax-M2' }}
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
@@ -181,18 +161,12 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# codex (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per #2578's lesson — empty key
|
||||
# silently falls through to the wrong SECRETS_JSON branch and
|
||||
# produces a confusing auth error 5 min later instead of the
|
||||
@@ -213,14 +187,10 @@ jobs:
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
codex|hermes)
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
google-adk)
|
||||
required_secret_name="MOLECULE_STAGING_GOOGLE_API_KEY"
|
||||
required_secret_value="${E2E_GOOGLE_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
|
||||
@@ -37,7 +37,6 @@ jobs:
|
||||
name: Intentional-failure teardown sanity
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 20
|
||||
|
||||
@@ -47,11 +46,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
E2E_MODE: smoke
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
@@ -66,12 +60,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken.
|
||||
|
||||
@@ -7,11 +7,10 @@
|
||||
# PR_NUMBER — set via ${{ github.event.pull_request.number }} from the trigger
|
||||
# POST_COMMENT — "true" to post/update comment on PR
|
||||
#
|
||||
# Gating logic (MVP signals 1,2,3,4,6):
|
||||
# Gating logic (MVP signals 1,2,3,6):
|
||||
# 1. Author-aware agent-tag comment scan
|
||||
# 2. REQUEST_CHANGES reviews state machine
|
||||
# 3. Staleness detection (SOP-12: review.commit_id != PR.head_sha + >1 working day)
|
||||
# 4. Branch divergence / scope-creep guard (base-sha vs target HEAD; mc#365)
|
||||
# 6. CI required-checks awareness
|
||||
#
|
||||
# Exit code: 0=CLEAR, 1=BLOCKED, 2=ERROR
|
||||
@@ -33,24 +32,6 @@ on:
|
||||
# iterating all open PRs when PR_NUMBER is empty.
|
||||
workflow_dispatch:
|
||||
|
||||
# Serialize per PR (or per repo for schedule/manual ticks) to prevent
|
||||
# the fan-out OOM class documented in
|
||||
# `reference_operator_host_python3_oom_storm_2026_05_18`. `edited`
|
||||
# events fan out on every PR-body edit; combined with the hourly cron
|
||||
# and synchronize bursts this workflow can stack runs of the same
|
||||
# workflow_id on the same PR (each ~4GB anon-RSS) and trip the
|
||||
# `--memory=4g --memory-swap=8g` per-container cap.
|
||||
#
|
||||
# NO `cancel-in-progress` (defaults to false). Per
|
||||
# `feedback_janitor_supersede_must_group_by_workflow_id`, cancelling
|
||||
# in-flight runs of any required-check-shaped workflow risks the
|
||||
# dismiss_stale_approvals + empty-commit-rerun dance (Gitea 1.22.6 has
|
||||
# no REST rerun). The gate-check is `continue-on-error: true` +
|
||||
# idempotent (POST/PATCH gate-check comment by context) so sequential
|
||||
# ticks are strictly safe.
|
||||
concurrency:
|
||||
group: gate-check-v3-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
|
||||
|
||||
permissions:
|
||||
# read: contents — for checkout (base ref, not PR head for security)
|
||||
# read: pull-requests — for reading PR info via API
|
||||
@@ -63,10 +44,8 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: PR advisory bot; merge blocking is enforced by CI status and branch protection.
|
||||
gate-check:
|
||||
runs-on: ubuntu-latest
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true # Never block on our own detector failing
|
||||
steps:
|
||||
- name: Check out BASE ref (never PR-head under pull_request_target)
|
||||
@@ -83,7 +62,6 @@ jobs:
|
||||
if: github.event_name == 'pull_request_target' || github.event.inputs.pr_number != ''
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
|
||||
POST_COMMENT: ${{ github.event.inputs.post_comment || 'true' }}
|
||||
run: |
|
||||
@@ -98,49 +76,25 @@ jobs:
|
||||
if: github.event_name == 'schedule'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fetch all open PRs and run gate-check on each. This scheduled
|
||||
# refresher is advisory; a transient Gitea list timeout must not turn
|
||||
# main red. PR-specific gate-check runs still use normal failure
|
||||
# semantics.
|
||||
pr_numbers=$(python3 <<'PY'
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
socket.setdefaulttimeout(30)
|
||||
token = os.environ["GITEA_TOKEN"]
|
||||
repo = os.environ["REPO"]
|
||||
url = f"https://git.moleculesai.app/api/v1/repos/{repo}/pulls?state=open&limit=100"
|
||||
last_error = None
|
||||
for attempt in range(1, 4):
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"Authorization": f"token {token}", "Accept": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
prs = json.loads(r.read())
|
||||
break
|
||||
except (TimeoutError, OSError, urllib.error.URLError, urllib.error.HTTPError) as exc:
|
||||
last_error = exc
|
||||
print(f"warning: PR list fetch attempt {attempt}/3 failed: {exc}", file=sys.stderr)
|
||||
if attempt < 3:
|
||||
time.sleep(2 * attempt)
|
||||
else:
|
||||
print(f"warning: skipped scheduled gate-check refresh; failed to list open PRs after 3 attempts: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(0)
|
||||
for pr in prs:
|
||||
print(pr["number"])
|
||||
PY
|
||||
)
|
||||
# Fetch all open PRs and run gate-check on each
|
||||
# socket.setdefaulttimeout(15): defence-in-depth for missing SOP_TIER_CHECK_TOKEN.
|
||||
# gate_check.py uses timeout=15 on every urlopen call; this catches the
|
||||
# inline Python polling loop too (issue #603).
|
||||
pr_numbers=$(python3 -c "
|
||||
import socket, urllib.request, json, os
|
||||
socket.setdefaulttimeout(15)
|
||||
token = os.environ['GITEA_TOKEN']
|
||||
req = urllib.request.Request(
|
||||
'https://git.moleculesai.app/api/v1/repos/${{ github.repository }}/pulls?state=open&limit=100',
|
||||
headers={'Authorization': f'token {token}', 'Accept': 'application/json'}
|
||||
)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
prs = json.loads(r.read())
|
||||
for pr in prs:
|
||||
print(pr['number'])
|
||||
")
|
||||
for pr in $pr_numbers; do
|
||||
echo "Checking PR #$pr..."
|
||||
python3 tools/gate-check-v3/gate_check.py \
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
name: gitea-merge-queue
|
||||
|
||||
# External serialized merge queue for Gitea 1.22.6.
|
||||
#
|
||||
# Gitea's `pull_auto_merge` table is not a real merge queue: it does not
|
||||
# serialize green PRs against a freshly-tested latest main. This workflow runs
|
||||
# the user-space queue bot, one PR per tick, using the non-bypass merge actor.
|
||||
#
|
||||
# Queue contract:
|
||||
# - add label `merge-queue` to an open same-repo PR
|
||||
# - bot updates stale PR heads with current main, then waits for CI
|
||||
# - bot merges only when current main is green and required PR contexts pass
|
||||
# - add `merge-queue-hold` to pause a queued PR without removing it
|
||||
|
||||
on:
|
||||
# Schedule moved to operator-config:
|
||||
# /etc/cron.d/molecule-core-merge-queue ->
|
||||
# /usr/local/bin/molecule-core-cron-bot.sh merge-queue
|
||||
#
|
||||
# The queue bot still processes one PR per tick, but no longer occupies
|
||||
# one of the shared Actions runners just to poll.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: gitea-merge-queue-${{ github.repository }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
queue:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out queue script from main
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Process one queued PR
|
||||
env:
|
||||
# AUTO_SYNC_TOKEN is the devops-engineer persona PAT. It is the
|
||||
# non-bypass merge actor allowed by branch protection.
|
||||
GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
WATCH_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
HOLD_LABEL: merge-queue-hold
|
||||
UPDATE_STYLE: merge
|
||||
REQUIRED_CONTEXTS: >-
|
||||
CI / all-required (pull_request),
|
||||
sop-checklist / all-items-acked (pull_request)
|
||||
# Push-side required contexts. Checking CI / all-required (push)
|
||||
# explicitly instead of the combined state avoids false-pause when
|
||||
# non-blocking jobs (continue-on-error: true) have failed — those
|
||||
# failures pollute combined state but do not gate merges.
|
||||
# NOTE: the event-suffixed context name is intentional — branch protection
|
||||
# MUST require `CI / all-required (pull_request)` (with suffix), NOT the
|
||||
# bare `CI / all-required`. Gitea treats absent contexts as pending, not
|
||||
# skipped; requiring the bare name silently blocks all merges (issue #1473).
|
||||
PUSH_REQUIRED_CONTEXTS: CI / all-required (push)
|
||||
run: python3 .gitea/scripts/gitea-merge-queue.py
|
||||
@@ -77,37 +77,39 @@ env:
|
||||
jobs:
|
||||
detect-changes:
|
||||
name: detect-changes
|
||||
# mc#1529 §1: pin to `docker-host` so the integration job runs on the
|
||||
# operator-host runners (molecule-runner-*), which carry the
|
||||
# `molecule-core-net` bridge network this workflow depends on. PC2
|
||||
# runners (hongming-pc-runner-*) also advertise ubuntu-latest but
|
||||
# don't have that network — the previous `runs-on: ubuntu-latest`
|
||||
# rolled the dice and hard-failed the bridge-inspect step ~30% of
|
||||
# the time. detect-changes itself doesn't need the bridge, but keeping
|
||||
# both jobs on the same label avoids workspace-volume cross-host
|
||||
# surprises and keeps the routing rule discoverable in one place.
|
||||
runs-on: docker-host
|
||||
# mc#1982 Phase 3 (RFC §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
handlers: ${{ steps.filter.outputs.handlers }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# A full-history checkout can exceed the runner's quiet/startup
|
||||
# window before the path filter emits logs. Fetch the common push
|
||||
# case cheaply; the script below fetches the exact BASE SHA if it is
|
||||
# not present in the shallow checkout.
|
||||
fetch-depth: 2
|
||||
fetch-depth: 0
|
||||
- id: filter
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile handlers-postgres \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "${{ github.event.pull_request.base.sha }}" \
|
||||
--base-ref "${{ github.event.pull_request.base.ref }}" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-}"
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/internal/handlers/|workspace-server/internal/wsauth/|workspace-server/migrations/|\.gitea/workflows/handlers-postgres-integration\.yml$)'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "handlers=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Single-job-with-per-step-if pattern: always runs to satisfy the
|
||||
# required-check name on branch protection; real work gates on the
|
||||
@@ -115,11 +117,8 @@ jobs:
|
||||
integration:
|
||||
name: Handlers Postgres Integration
|
||||
needs: detect-changes
|
||||
# mc#1529 §1: must run on operator-host (where `molecule-core-net`
|
||||
# exists). See detect-changes for the full routing rationale.
|
||||
runs-on: docker-host
|
||||
# mc#1982 Phase 3 (RFC §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
env:
|
||||
# Unique name per run so concurrent jobs don't collide on the
|
||||
|
||||
@@ -60,17 +60,9 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: change detector only; downstream Harness Replays is the meaningful gate.
|
||||
detect-changes:
|
||||
# mc#1529 follow-on: pin to `docker-host` so this lane lands on
|
||||
# Linux operator-host runners (the only ones with a working
|
||||
# docker.sock + `molecule-core-net`). The bare `ubuntu-latest`
|
||||
# label is also matched by hongming-pc-runner-* (Windows act_runner
|
||||
# v1.0.3), where the `docker compose ...` exec below fails. Mirror
|
||||
# of mc#1543; see internal#512 for class defect.
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
run: ${{ steps.decide.outputs.run }}
|
||||
@@ -139,14 +131,7 @@ jobs:
|
||||
RESP=$(curl -sS --fail --max-time 30 \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Accept: application/json" \
|
||||
"$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/compare/$BASE...$HEAD") || {
|
||||
# If Gitea's Compare API is slow/unavailable, choose the conservative
|
||||
# behavior: run the harness instead of failing the detector and polluting
|
||||
# main with a red non-gate context.
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=compare-api-unavailable base=$BASE head=$HEAD" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
}
|
||||
"$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/compare/$BASE...$HEAD")
|
||||
DIFF_FILES=$(echo "$RESP" | bash .gitea/scripts/compare-api-diff-files.py 2>/dev/null || true)
|
||||
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF_FILES" >> "$GITHUB_OUTPUT"
|
||||
@@ -164,15 +149,11 @@ jobs:
|
||||
# matches e2e-api.yml — see that workflow's comment for why a
|
||||
# job-level `if: false` would block branch protection via the
|
||||
# SKIPPED-in-set bug.
|
||||
# bp-exempt: path-filtered replay suite; CI / all-required is the branch-protection aggregate.
|
||||
harness-replays:
|
||||
needs: detect-changes
|
||||
name: Harness Replays
|
||||
# mc#1529 follow-on: `docker compose ... ps/logs` against tenant-alpha/
|
||||
# beta containers. Must run on operator-host Linux (docker-host).
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
name: lint-bp-context-emit-match
|
||||
|
||||
# Tier 2f scheduled lint (per mc#1982) — detects drift between
|
||||
# `branch_protections/<branch>.status_check_contexts` and the set of
|
||||
# contexts emitted by `.gitea/workflows/*.yml`.
|
||||
#
|
||||
# Rule
|
||||
# ----
|
||||
# For each protected branch context (Source A — BP), there must exist
|
||||
# at least one emitting workflow + job pair (Source B — workflow YAML
|
||||
# + on:-event mapping) whose runtime status-name maps to it. The
|
||||
# inverse direction (emitter without BP context) is informational
|
||||
# only — Tier 2g handles that at PR-time.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# A BP-required context with no emitter blocks merges forever — Gitea
|
||||
# 1.22.6 treats absent-as-`pending`, NOT absent-as-`skipped`. The
|
||||
# phantom-required-check class previously surfaced as
|
||||
# `feedback_phantom_required_check_after_gitea_migration` (a port
|
||||
# kept the GitHub context name after rename to Gitea, but no
|
||||
# workflow emitted under the new name).
|
||||
#
|
||||
# This lint catches the same class structurally + a forward case:
|
||||
# workflow renamed/deleted while still in BP.
|
||||
#
|
||||
# Scope
|
||||
# -----
|
||||
# Scheduled daily. We DON'T run on `pull_request` because (a) the
|
||||
# emitter side moves with PR diffs (transitional state false-flags)
|
||||
# and (b) Tier 2g handles emitter-side drift at PR-time.
|
||||
#
|
||||
# Cross-repo
|
||||
# ----------
|
||||
# Today this runs only on molecule-core/main. Per internal#349
|
||||
# (cross-repo BP sweep) Class-D repos will get the same lint after
|
||||
# their BP rollouts.
|
||||
#
|
||||
# Auth
|
||||
# ----
|
||||
# `GET /repos/.../branch_protections/{branch}` requires repo-admin
|
||||
# role on Gitea 1.22.6. We use DRIFT_BOT_TOKEN (same persona as
|
||||
# ci-required-drift.yml — `internal#329` provisioning trail).
|
||||
# Graceful-degrade per Tier 2a contract: 403/404 → exit 0 with
|
||||
# ::error::.
|
||||
#
|
||||
# Idempotency
|
||||
# -----------
|
||||
# The drift issue is filed with title prefix
|
||||
# `[ci-bp-drift] {repo}/{branch}: BP→emitter mismatch`. The script
|
||||
# searches OPEN issues for an exact title-prefix match and PATCHes
|
||||
# the existing issue (if any) instead of POSTing a duplicate.
|
||||
# Mirrors `ci-required-drift.py`'s contract.
|
||||
#
|
||||
# Phase contract (RFC internal#219 §1 ladder)
|
||||
# -------------------------------------------
|
||||
# Lands at `continue-on-error: true` (Phase 3). After 7 days of clean
|
||||
# scheduled runs on `main`, flip to `false` so a scheduled failure
|
||||
# becomes a hard CI signal.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#1982 (the RFC that specs this lint)
|
||||
# - internal#349 (cross-repo BP sweep)
|
||||
# - feedback_phantom_required_check_after_gitea_migration
|
||||
# - feedback_tier_label_ids_are_per_repo
|
||||
# - ci-required-drift.yml (F2 detector, narrower-scope sibling)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Daily at 03:31 UTC — off-peak, prime-staggered from other
|
||||
# scheduled jobs (ci-required-drift :00 hourly, lint-coe-tracking
|
||||
# 13:11). At 03:31 the CI fleet is quietest in EMEA hours.
|
||||
- cron: '31 3 * * *'
|
||||
workflow_dispatch:
|
||||
# No `push` / `pull_request` here — Tier 2g owns PR-time drift.
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write # needed to file/edit the drift issue
|
||||
|
||||
concurrency:
|
||||
group: lint-bp-context-emit-match-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: lint-bp-context-emit-match
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# Phase 4 (RFC #219 §1): 22 days green since 2026-05-11 port,
|
||||
# well past the 7-clean-run threshold. Scheduled failure is now
|
||||
# a hard CI signal.
|
||||
continue-on-error: false
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Run lint-bp-context-emit-match
|
||||
env:
|
||||
# DRIFT_BOT_TOKEN — repo-admin on this repo (internal#329
|
||||
# provisioning trail). Required for branch_protections read.
|
||||
GITEA_TOKEN: ${{ secrets.DRIFT_BOT_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
BRANCH: main
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
DRIFT_LABEL: ci-bp-drift
|
||||
GITHUB_RUN_URL: https://git.moleculesai.app/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: python3 .gitea/scripts/lint_bp_context_emit_match.py
|
||||
- name: Run lint-bp-context-emit-match unit tests
|
||||
run: |
|
||||
python -m pip install --quiet pytest
|
||||
python3 -m pytest tests/test_lint_bp_context_emit_match.py -v
|
||||
@@ -1,6 +1,6 @@
|
||||
name: lint-continue-on-error-tracking
|
||||
|
||||
# Tier 2e hard-gate lint (per mc#1982) — every
|
||||
# Tier 2e hard-gate lint (per internal#350) — every
|
||||
# `continue-on-error: true` in `.gitea/workflows/*.yml` must carry a
|
||||
# `# mc#NNNN` or `# internal#NNNN` tracker comment within 2 lines,
|
||||
# the referenced issue must be OPEN, and ≤14 days old.
|
||||
@@ -8,7 +8,7 @@ name: lint-continue-on-error-tracking
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# `continue-on-error: true` on `platform-build` had been hiding
|
||||
# mc#1982-class regressions for ~3 weeks before #656 surfaced them on
|
||||
# mc#664-class regressions for ~3 weeks before #656 surfaced them on
|
||||
# 2026-05-12. A 14-day cap on tracker age forces a review cycle and
|
||||
# surfaces mask-drift within at most 14 days of the original defect.
|
||||
# Each `continue-on-error: true` gets a paper trail — close or renew.
|
||||
@@ -45,12 +45,12 @@ name: lint-continue-on-error-tracking
|
||||
# close-and-flip, or document the deliberate keep-mask in a fresh
|
||||
# 14-day-renewable tracker. After main is clean for 3 days,
|
||||
# follow-up PR flips this workflow's continue-on-error to false.
|
||||
# Tracking: mc#1982.
|
||||
# Tracking: internal#350.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#1982 (the RFC that specs this lint)
|
||||
# - mc#1982 (the empirical masked-3-weeks case)
|
||||
# - internal#350 (the RFC that specs this lint)
|
||||
# - mc#664 (the empirical masked-3-weeks case)
|
||||
# - feedback_chained_defects_in_never_tested_workflows
|
||||
# - feedback_behavior_based_ast_gates
|
||||
# - feedback_strict_root_only_after_class_a
|
||||
@@ -89,17 +89,15 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: meta-lint for masked jobs; tracked separately until masks are burned down.
|
||||
lint:
|
||||
name: lint-continue-on-error-tracking
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 10
|
||||
# Phase 3 (RFC #219 §1): surface masked defects without blocking
|
||||
# PRs. Pre-existing continue-on-error: true directives on main
|
||||
# all violate this lint at first — intentional. Flip to false
|
||||
# follow-up after main is clean for 3 days. mc#1982.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true # mc#1982 Phase 3 mask — 14d forced-renewal cadence
|
||||
# follow-up after main is clean for 3 days. internal#350.
|
||||
continue-on-error: true # internal#350 Phase 3 mask — 14d forced-renewal cadence
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
|
||||
@@ -30,16 +30,10 @@ name: Lint curl status-code capture
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint-curl-status-capture.py'
|
||||
- 'tests/test_lint_curl_status_capture.py'
|
||||
paths: ['.gitea/workflows/**']
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint-curl-status-capture.py'
|
||||
- 'tests/test_lint_curl_status_capture.py'
|
||||
paths: ['.gitea/workflows/**']
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
@@ -48,11 +42,63 @@ jobs:
|
||||
scan:
|
||||
name: Scan workflows for curl status-capture pollution
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 4 (RFC #219 §1): 22 days green since 2026-05-11 port.
|
||||
# mc#1982 mask removed — no surfaced defects in this lane.
|
||||
continue-on-error: false
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
|
||||
run: |
|
||||
python3 .gitea/scripts/lint-curl-status-capture.py
|
||||
set -uo pipefail
|
||||
# Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
|
||||
# subshell where the entire command-substitution wraps a curl that
|
||||
# ends with `|| echo "000"`. Must distinguish from the SAFE shape
|
||||
# `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
|
||||
# tempfile produces empty stdout, no pollution.
|
||||
python3 <<'PY'
|
||||
import os, re, sys, glob
|
||||
|
||||
BAD_FILES = []
|
||||
|
||||
# Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
|
||||
# The `\\n` is the bash line-continuation that lets curl flags span lines.
|
||||
# We collapse continuation lines first, then look for the single-line bad pattern.
|
||||
PATTERN = re.compile(
|
||||
r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Self-skip: this lint workflow contains the literal anti-pattern in
|
||||
# its own docstring — that's intentional, not a bug.
|
||||
SELF = ".gitea/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
for f in sorted(glob.glob(".gitea/workflows/*.yml")):
|
||||
if f == SELF:
|
||||
continue
|
||||
with open(f) as fh:
|
||||
content = fh.read()
|
||||
# Collapse bash line-continuations (\\\n + leading whitespace)
|
||||
# into a single logical line so the regex can see the full
|
||||
# curl invocation as one chunk.
|
||||
flat = re.sub(r'\\\s*\n\s*', ' ', content)
|
||||
for m in PATTERN.finditer(flat):
|
||||
BAD_FILES.append((f, m.group(0)[:120]))
|
||||
|
||||
if not BAD_FILES:
|
||||
print("OK No curl-status-capture pollution patterns detected")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
|
||||
for f, snippet in BAD_FILES:
|
||||
print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
|
||||
print(f" matched: {snippet}...")
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(' set +e')
|
||||
print(' curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
|
||||
print(' set -e')
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
sys.exit(1)
|
||||
PY
|
||||
|
||||
@@ -1,306 +0,0 @@
|
||||
name: Lint forbidden tenant-env keys
|
||||
|
||||
# RFC#523 Layer 3 (task #146): scan workspace_secrets-writer Go code
|
||||
# under workspace-server/ for new code that hardcodes a forbidden
|
||||
# operator-scope env var NAME (GITEA_TOKEN, CP_ADMIN_API_TOKEN,
|
||||
# RAILWAY_TOKEN, INFISICAL_OPERATOR_TOKEN, MOLECULE_OPERATOR_*, …).
|
||||
#
|
||||
# Catches the class "a new writer accidentally widens the propagation
|
||||
# set" — e.g. a future env-mutator plugin that sets envVars["GITEA_TOKEN"]
|
||||
# directly. Today the L1 runtime guard would abort the provision, but
|
||||
# this lint surfaces the offending code at PR review time instead of
|
||||
# at first provision attempt.
|
||||
#
|
||||
# Companion layers:
|
||||
# - L1: workspace-server/internal/handlers/workspace_provision_forbidden_env.go
|
||||
# (fail-closed abort at provision time)
|
||||
# - L2: workspace/entrypoint.sh top-of-file env-grep + exit 1
|
||||
#
|
||||
# Open-source-template-friendly: the deny pattern is generic. A fork
|
||||
# can copy this workflow and replace OPERATOR_KEY_PATTERN with its
|
||||
# own operator-scope key names.
|
||||
#
|
||||
# Path-filter discipline:
|
||||
# This workflow runs on every PR (no paths: filter — see
|
||||
# feedback_path_filtered_workflow_cant_be_required). The scan itself
|
||||
# targets workspace_secrets-writer paths via grep -r; it's fast
|
||||
# (sub-second) so unconditional run is fine.
|
||||
#
|
||||
# ── 2026-06-01 CI-scheduler-fanout consolidation (fix/ci-scheduler-fanout) ──
|
||||
# The RFC#523 sibling lint formerly in its own file
|
||||
# `lint-no-tenant-gitea-token.yml` (the broader "no repo-host token into
|
||||
# any tenant-writer surface" scan) is now a SECOND job in THIS workflow
|
||||
# (`scan-tenant-token-write`). Both are sub-second Go-source greps that
|
||||
# fired as two separate workflow runs on every PR — pure scheduler
|
||||
# fan-out. Folding the sibling in here drops one workflow run + one
|
||||
# checkout per PR while keeping BOTH scans firing unconditionally on
|
||||
# every PR (the no-paths discipline above is preserved — neither job is
|
||||
# paths-filtered). The moved job keeps its exact `name:` so its emitted
|
||||
# status context is unchanged in substance; its `# bp-exempt:` directive
|
||||
# moves with it (Tier 2g). The old `Lint no tenant GITEA or GITHUB token
|
||||
# write / …` context is retired (a disappearing context needs no
|
||||
# directive; only NEW emitters do).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan workspace_secrets writers for forbidden env keys
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Scan for forbidden operator-scope env key NAMES in writer paths
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Forbidden EXACT-MATCH env var names. Kept in lockstep with
|
||||
# workspace-server/internal/handlers/workspace_provision_forbidden_env.go
|
||||
# forbiddenTenantEnvKeys. The Go-side test
|
||||
# TestIsForbiddenTenantEnvKey_ExactMatches is the source of
|
||||
# truth — if Go-side adds a key, also add it here (and
|
||||
# vice-versa). Drift between the two is the failure mode this
|
||||
# entire 3-layer guardrail is designed to catch.
|
||||
FORBIDDEN_KEYS=(
|
||||
"GITEA_TOKEN" "GITEA_PAT"
|
||||
"GITHUB_TOKEN" "GITHUB_PAT" "GH_TOKEN"
|
||||
"GITLAB_TOKEN" "GL_TOKEN"
|
||||
"BITBUCKET_TOKEN"
|
||||
"CP_ADMIN_API_TOKEN" "CP_ADMIN_TOKEN"
|
||||
"INFISICAL_OPERATOR_TOKEN" "INFISICAL_BOOTSTRAP_TOKEN"
|
||||
"RAILWAY_TOKEN" "RAILWAY_PERSONAL_API_TOKEN"
|
||||
"HETZNER_TOKEN" "HETZNER_API_TOKEN"
|
||||
)
|
||||
|
||||
# Forbidden PREFIX patterns — operator-scope families.
|
||||
FORBIDDEN_PREFIXES=(
|
||||
"MOLECULE_OPERATOR_"
|
||||
)
|
||||
|
||||
# Writer paths: Go source under workspace-server/ that
|
||||
# writes to the env-vars map or to workspace_secrets DB rows.
|
||||
# Tests, the forbidden-env source itself, and the silent-
|
||||
# strip denylist are exempt (they LIST the keys by design).
|
||||
SCAN_ROOT="workspace-server/internal"
|
||||
# Exempt paths fall in two classes:
|
||||
# 1. The deny-set definitions + the silent-strip denylist:
|
||||
# they LIST the forbidden names by design.
|
||||
# 2. Pre-RFC#523 persona-merge / config-read paths that
|
||||
# already handle these names correctly (the silent-
|
||||
# strip downstream + the new L1 fail-closed cover the
|
||||
# runtime risk; these reads are unchanged).
|
||||
# New code MUST NOT be added to this list without reviewer
|
||||
# signoff and a one-line justification in this diff.
|
||||
EXEMPT_PATHS=(
|
||||
# Class 1 — deny-set definitions
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env.go"
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env_test.go"
|
||||
"workspace-server/internal/provisioner/provisioner.go"
|
||||
"workspace-server/internal/provisioner/provisioner_test.go"
|
||||
# Class 2 — pre-existing persona-fallback / org-helper paths
|
||||
# that set the GITEA_TOKEN fallback lane (stripped downstream
|
||||
# by provisioner.buildContainerEnv per forensic #145). The
|
||||
# new L1 fail-closed runs BEFORE these writers, so any
|
||||
# operator-scope leak via global/workspace_secrets is
|
||||
# already caught. See applyAgentGitHTTPCreds doc-comment.
|
||||
"workspace-server/internal/handlers/agent_git_identity.go"
|
||||
"workspace-server/internal/handlers/org_helpers.go"
|
||||
"workspace-server/internal/handlers/org.go"
|
||||
# Class 2 — CP→platform admin auth (NOT a tenant env write;
|
||||
# this is the control-plane HTTP auth header source).
|
||||
"workspace-server/internal/provisioner/cp_provisioner.go"
|
||||
)
|
||||
|
||||
# Build a single grep -F pattern: every forbidden key wrapped
|
||||
# in quotes (Go string-literal form, which is how env-map
|
||||
# writes appear). e.g. envVars["GITEA_TOKEN"] = ... or
|
||||
# `"GITEA_TOKEN":` in a literal-map declaration.
|
||||
#
|
||||
# We deliberately match the quoted form so a comment that
|
||||
# happens to spell the name without quotes (e.g. "see
|
||||
# GITEA_TOKEN below") doesn't trip the lint.
|
||||
PATTERN=""
|
||||
for k in "${FORBIDDEN_KEYS[@]}"; do
|
||||
PATTERN="${PATTERN}\"${k}\"\n"
|
||||
done
|
||||
for p in "${FORBIDDEN_PREFIXES[@]}"; do
|
||||
# Prefix match needs a regex; switch to grep -E below for
|
||||
# this slice. Kept conceptually here so the deny set lives
|
||||
# in one place; scan is run twice (literal + prefix).
|
||||
true
|
||||
done
|
||||
|
||||
# Build exempt-paths grep filter — `grep -v -f` style.
|
||||
EXEMPT_FILTER=$(mktemp)
|
||||
trap 'rm -f "$EXEMPT_FILTER"' EXIT
|
||||
for p in "${EXEMPT_PATHS[@]}"; do
|
||||
echo "$p" >> "$EXEMPT_FILTER"
|
||||
done
|
||||
|
||||
# --- Exact-match scan ---
|
||||
HITS=""
|
||||
for k in "${FORBIDDEN_KEYS[@]}"; do
|
||||
# Only .go files; skip _test.go for the writer-path scan
|
||||
# since tests legitimately reference the names. The
|
||||
# writer-path lint targets PRODUCTION code only.
|
||||
found=$(grep -rn --include='*.go' --exclude='*_test.go' "\"${k}\"" "$SCAN_ROOT" 2>/dev/null \
|
||||
| grep -v -F -f "$EXEMPT_FILTER" || true)
|
||||
if [ -n "$found" ]; then
|
||||
HITS="${HITS}${found}\n"
|
||||
fi
|
||||
done
|
||||
|
||||
# --- Prefix scan ---
|
||||
for prefix in "${FORBIDDEN_PREFIXES[@]}"; do
|
||||
found=$(grep -rnE --include='*.go' --exclude='*_test.go' "\"${prefix}[A-Z0-9_]+\"" "$SCAN_ROOT" 2>/dev/null \
|
||||
| grep -v -F -f "$EXEMPT_FILTER" || true)
|
||||
if [ -n "$found" ]; then
|
||||
HITS="${HITS}${found}\n"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$HITS" ]; then
|
||||
echo "::error::RFC#523 Layer 3: forbidden operator-scope env var name(s) hardcoded in tenant-workspace writer paths:"
|
||||
printf "$HITS"
|
||||
echo ""
|
||||
echo "These env-var NAMES are on the operator-scope deny list (see"
|
||||
echo "workspace-server/internal/handlers/workspace_provision_forbidden_env.go)."
|
||||
echo "If your code legitimately needs to inject one of these for a"
|
||||
echo "non-tenant code path, add the file to EXEMPT_PATHS in this"
|
||||
echo "workflow with a one-line justification — reviewer signoff required."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK No forbidden operator-scope env key names hardcoded in writer paths."
|
||||
|
||||
# bp-exempt: advisory RFC#523 lint; PR review gate is review-driven, not BP-driven.
|
||||
# (Carried with the workflow-name rename in PR mc#1593 so the renamed
|
||||
# context emission satisfies lint_required_context_exists_in_bp Tier 2g.)
|
||||
scan-tenant-token-write:
|
||||
name: Scan for repo-host token write into tenant workspace surface
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Find Go files referencing a tenant-writer surface AND a repo-host token
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Repo-host token NAMES — the threat-model subset. Operator-fleet
|
||||
# tokens (CP_ADMIN_API_TOKEN, RAILWAY_TOKEN, INFISICAL_*) are
|
||||
# caught by lint-forbidden-env-keys.yml's broader deny set; this
|
||||
# lint focuses on the git-host class so a single co-occurrence
|
||||
# match has a low false-positive rate.
|
||||
FORBIDDEN_KEYS=(
|
||||
"GITEA_TOKEN"
|
||||
"GITEA_PAT"
|
||||
"GITHUB_TOKEN"
|
||||
"GITHUB_PAT"
|
||||
"GH_TOKEN"
|
||||
)
|
||||
|
||||
# Tenant-writer surface markers. A file matches the surface set
|
||||
# if it references ANY of these strings. This is the "is this
|
||||
# code path writing into a tenant workspace?" heuristic.
|
||||
# Curated to catch the actual code shapes used in this repo
|
||||
# (verified by grep against current main 2026-05-19):
|
||||
# - "workspace_secrets" / "global_secrets" → DB table writes
|
||||
# - "seedAllowList" → CP-side seed table
|
||||
# - "/settings/secrets" → tenant HTTP API write
|
||||
# - "envVars[" → in-memory env map write
|
||||
# - "containerEnv" → docker-run env-set
|
||||
# - "userData" → EC2 user-data script
|
||||
# - "provisionPayload" / "provisionContext" → provision-request shape
|
||||
SURFACE_PATTERN='workspace_secrets|global_secrets|seedAllowList|/settings/secrets|envVars\[|containerEnv|userData|provisionPayload|provisionContext'
|
||||
|
||||
# Files that legitimately reference these names AND a surface
|
||||
# marker, but do so for guard / strip / test / doc-comment
|
||||
# reasons. New entries require reviewer signoff and a one-line
|
||||
# justification in the diff.
|
||||
EXEMPT_FILES=(
|
||||
# RFC#523 L1 deny-set source-of-truth + tests
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env.go"
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env_test.go"
|
||||
# Forensic-#145 silent-strip denylist (defense-in-depth, by design lists the names)
|
||||
"workspace-server/internal/provisioner/provisioner.go"
|
||||
"workspace-server/internal/provisioner/provisioner_test.go"
|
||||
# Pre-RFC#523 persona-fallback / org-helper paths. The L1
|
||||
# fail-closed runs BEFORE these writers; downstream silent-strip
|
||||
# also covers them. See applyAgentGitHTTPCreds doc-comment.
|
||||
"workspace-server/internal/handlers/agent_git_identity.go"
|
||||
"workspace-server/internal/handlers/org_helpers.go"
|
||||
"workspace-server/internal/handlers/org.go"
|
||||
# CP→platform admin auth (NOT a tenant env write).
|
||||
"workspace-server/internal/provisioner/cp_provisioner.go"
|
||||
)
|
||||
|
||||
# Build an extended-regex alternation of forbidden keys.
|
||||
KEY_ALT="$(IFS='|'; echo "${FORBIDDEN_KEYS[*]}")"
|
||||
|
||||
# Find candidate files: Go non-test sources that contain a
|
||||
# tenant-writer surface marker.
|
||||
mapfile -t CANDIDATES < <(
|
||||
grep -rlE --include='*.go' --exclude='*_test.go' \
|
||||
"${SURFACE_PATTERN}" . 2>/dev/null \
|
||||
| sed 's|^\./||' \
|
||||
| sort -u
|
||||
)
|
||||
|
||||
if [ "${#CANDIDATES[@]}" -eq 0 ]; then
|
||||
echo "OK No tenant-writer-surface files found in tree (unexpected, but not a lint failure)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
HITS=""
|
||||
for f in "${CANDIDATES[@]}"; do
|
||||
# Skip exempt files.
|
||||
skip=0
|
||||
for ex in "${EXEMPT_FILES[@]}"; do
|
||||
if [ "$f" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ "$skip" = "1" ] && continue
|
||||
|
||||
# File contains a surface marker; now grep for a forbidden
|
||||
# key NAME. We require a QUOTED-literal match to avoid
|
||||
# firing on a comment like "// also handle GITEA_TOKEN".
|
||||
#
|
||||
# The literal form catches:
|
||||
# - os.Getenv("GITEA_TOKEN")
|
||||
# - envVars["GITEA_TOKEN"] = ...
|
||||
# - {envKey: "GITEA_TOKEN", tenantKey: "GITEA_TOKEN"}
|
||||
# but not:
|
||||
# - // see GITEA_TOKEN below (no quotes)
|
||||
found=$(grep -nE "\"(${KEY_ALT})\"" "$f" 2>/dev/null || true)
|
||||
if [ -n "$found" ]; then
|
||||
HITS="${HITS}--- ${f} ---\n${found}\n"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$HITS" ]; then
|
||||
echo "::error::Task #146 lint: repo-host token name(s) quoted in a tenant-writer-surface file:"
|
||||
printf "$HITS"
|
||||
echo ""
|
||||
echo "These files reference a tenant-writer surface (workspace_secrets,"
|
||||
echo "seedAllowList, /settings/secrets, containerEnv, userData, etc.)"
|
||||
echo "AND quote a repo-host token name (GITEA_TOKEN/GITHUB_TOKEN/…)."
|
||||
echo "Per RFC#523 threat model, tenant workspaces MUST NOT receive"
|
||||
echo "operator-scope repo-host tokens. If your code legitimately needs"
|
||||
echo "to reference one of these names in a tenant-writer file (e.g."
|
||||
echo "a deny-set definition or silent-strip list), add the file to"
|
||||
echo "EXEMPT_FILES with a one-line justification — reviewer signoff"
|
||||
echo "required."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK No tenant-writer-surface file co-mentions a repo-host token literal."
|
||||
@@ -1,6 +1,6 @@
|
||||
name: lint-mask-pr-atomicity
|
||||
|
||||
# Tier 2d hard-gate lint (per mc#1982) — blocks PRs that touch
|
||||
# Tier 2d hard-gate lint (per internal#350) — blocks PRs that touch
|
||||
# `.gitea/workflows/ci.yml` and modify ONLY ONE of {continue-on-error,
|
||||
# all-required.sentinel.needs} without a `Paired: #NNN` reference in
|
||||
# the PR body or in a commit message.
|
||||
@@ -37,13 +37,13 @@ name: lint-mask-pr-atomicity
|
||||
# This workflow lands at `continue-on-error: true` (Phase 3 — surface
|
||||
# regressions without blocking PRs while the rule beds in).
|
||||
# Follow-up PR flips to `false` once we have ≥3 days of clean runs on
|
||||
# `main` and no false-positives. Tracking issue: mc#1982.
|
||||
# `main` and no false-positives. Tracking issue: internal#350.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#1982 (the RFC that specs this lint)
|
||||
# - internal#350 (the RFC that specs this lint)
|
||||
# - PR#665 / PR#668 (the empirical split-pair)
|
||||
# - mc#1982 (the main-red incident the split caused)
|
||||
# - mc#664 (the main-red incident the split caused)
|
||||
# - feedback_strict_root_only_after_class_a
|
||||
# - feedback_behavior_based_ast_gates
|
||||
#
|
||||
@@ -84,7 +84,6 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: meta-lint advisory during mask burn-down; CI / all-required gates merges.
|
||||
scan:
|
||||
name: lint-mask-pr-atomicity
|
||||
runs-on: ubuntu-latest
|
||||
@@ -92,8 +91,7 @@ jobs:
|
||||
# Phase 3 (RFC #219 §1): surface broken shapes without blocking
|
||||
# PRs. Follow-up PR flips this to `false` once recent runs on main
|
||||
# are confirmed clean (eat-our-own-dogfood discipline mirrors
|
||||
# PR#673's same-shape comment). Tracking: mc#1982.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
# PR#673's same-shape comment). Tracking: internal#350.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Check out PR head with full history (need base SHA blobs)
|
||||
|
||||
@@ -4,7 +4,7 @@ name: Lint pre-flip continue-on-error
|
||||
# on any job in `.gitea/workflows/*.yml` WITHOUT proof that the affected
|
||||
# job's recent runs on the target branch (PR base) are actually green.
|
||||
#
|
||||
# Empirical class: PR #656 / mc#1982. PR #656 (RFC internal#219 Phase 4)
|
||||
# Empirical class: PR #656 / mc#664. PR #656 (RFC internal#219 Phase 4)
|
||||
# flipped 5 platform-build-class jobs `continue-on-error: true → false`
|
||||
# on the basis of a "verified green on main via combined-status check".
|
||||
# But that "green" was the LIE the prior `continue-on-error: true`
|
||||
@@ -13,7 +13,7 @@ name: Lint pre-flip continue-on-error
|
||||
# job-level status. The precondition the PR claimed to verify was
|
||||
# structurally fooled by the bug being flipped.
|
||||
#
|
||||
# mc#1982 captured the surfaced defects (2 mutually-masked regressions):
|
||||
# mc#664 captured the surfaced defects (2 mutually-masked regressions):
|
||||
# - Class 1: sqlmock helper drift since 2f36bb9a (24 days old)
|
||||
# - Class 2: OFFSEC-001 contract collision since 7d1a189f (1 day old)
|
||||
#
|
||||
@@ -55,7 +55,7 @@ name: Lint pre-flip continue-on-error
|
||||
# - YAML parse error in one of the workflow files: warn-only,
|
||||
# don't block — the YAML lint workflows catch this separately.
|
||||
#
|
||||
# Cross-links: PR#656, mc#1982, PR#665 (interim re-mask),
|
||||
# Cross-links: PR#656, mc#664, PR#665 (interim re-mask),
|
||||
# Quirk #10 (internal#342 + dup #287), hongming-pc2 charter
|
||||
# §SOP-N rule (e), feedback_strict_root_only_after_class_a,
|
||||
# feedback_no_shared_persona_token_use.
|
||||
@@ -99,8 +99,8 @@ jobs:
|
||||
timeout-minutes: 8
|
||||
# Phase 3 (RFC internal#219 §1): surface broken flips without blocking
|
||||
# the PR yet. Follow-up flips this to `false` once the workflow itself
|
||||
# has clean recent runs on main. mc#1982 interim — remove when CoE→false.
|
||||
continue-on-error: true # mc#1982
|
||||
# has clean recent runs on main. mc#664 interim — remove when CoE→false.
|
||||
continue-on-error: true # mc#664
|
||||
steps:
|
||||
- name: Check out PR head (full history for base-SHA access)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
name: lint-required-context-exists-in-bp
|
||||
|
||||
# Tier 2g hard-gate lint (per mc#1982) — diff-based PR-time
|
||||
# check. When a PR adds a NEW commit-status emission (workflow YAML
|
||||
# `name:` + job `name:`-or-key + on:-event), the workflow file must
|
||||
# carry one of three directives adjacent to the new job:
|
||||
#
|
||||
# - `# bp-required: yes` — and BP must list the context
|
||||
# - `# bp-required: pending #NNN` — acknowledged asymmetry + tracker
|
||||
# - `# bp-exempt: <reason>` — informational job, not a gate
|
||||
#
|
||||
# Default (no directive on a new emitter) = FAIL.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# PR#656 added `CI / all-required (pull_request)` as a sentinel
|
||||
# context that workflows emit, but BP did NOT list it. When
|
||||
# platform-build failed, all-required failed, but BP let the PR
|
||||
# merge anyway → cascade to mc#1982. With this lint, PR#656 would
|
||||
# have been blocked until either the BP PATCH ran alongside OR
|
||||
# the author added a `bp-required: pending` directive.
|
||||
#
|
||||
# Tier 2g vs Tier 2f
|
||||
# ------------------
|
||||
# Tier 2g runs at PR-time (diff-based) and BLOCKS the merge.
|
||||
# Tier 2f runs daily (scheduled) and FILES a drift issue. They
|
||||
# share the workflow-context enumeration helpers
|
||||
# (`_event_map`, `workflow_contexts`, `_job_display`) but the
|
||||
# semantics are intentionally distinct so they're separate scripts.
|
||||
# Co-design is documented in mc#1982.
|
||||
#
|
||||
# Directive comment lives in the workflow file (NOT PR body)
|
||||
# ----------------------------------------------------------
|
||||
# A PR-body claim of "BP exempt" evaporates on merge — the
|
||||
# asymmetry returns to undetected state and Tier 2f's daily
|
||||
# scheduled audit can't see it. The directive must live with the
|
||||
# emitter so both PR-time (Tier 2g) and post-merge (Tier 2f)
|
||||
# readers consume the same source.
|
||||
#
|
||||
# Phase contract (RFC internal#219 §1 ladder)
|
||||
# -------------------------------------------
|
||||
# Lands at `continue-on-error: true` (Phase 3 — surface the
|
||||
# pattern without blocking PRs while the directive convention
|
||||
# beds in). After 7 days of clean runs on `main` with no false
|
||||
# positives, follow-up flips to `false`. Tracking: mc#1982.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#1982 (the RFC that specs this lint)
|
||||
# - PR#656 (the empirical case)
|
||||
# - mc#1982 (the surfaced cascade)
|
||||
# - feedback_phantom_required_check_after_gitea_migration (Tier 2f cousin)
|
||||
# - feedback_behavior_based_ast_gates
|
||||
#
|
||||
# Auth: DRIFT_BOT_TOKEN (repo-admin for branch_protections read).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint_required_context_exists_in_bp.py'
|
||||
- '.gitea/workflows/lint-required-context-exists-in-bp.yml'
|
||||
- 'tests/test_lint_required_context_exists_in_bp.py'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lint-required-context-exists-in-bp-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: this lint is a PR-time advisory and is not intended to
|
||||
# be a required gate on main. The directive eat-our-own-dogfood
|
||||
# confirms the convention works on the lint that defines it.
|
||||
lint:
|
||||
name: lint-required-context-exists-in-bp
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# Phase 4 (RFC #219 §1): 22 days green since 2026-05-11 port,
|
||||
# well past the 7-clean-day threshold. PR-time failure is now
|
||||
# a hard CI signal.
|
||||
continue-on-error: false
|
||||
steps:
|
||||
- name: Check out PR head with full history (need base SHA blobs)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# `git show <base-sha>:<path>` needs the base SHA's blobs.
|
||||
# Same rationale as PR#673 and check-migration-collisions.yml.
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Ensure base ref is reachable locally
|
||||
# Cheap insurance against runner-version drift.
|
||||
run: |
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
- name: Run lint-required-context-exists-in-bp
|
||||
env:
|
||||
# DRIFT_BOT_TOKEN — repo-admin (needed for branch_protections).
|
||||
GITEA_TOKEN: ${{ secrets.DRIFT_BOT_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
BRANCH: main
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
run: python3 .gitea/scripts/lint_required_context_exists_in_bp.py
|
||||
- name: Run lint-required-context-exists-in-bp unit tests
|
||||
run: |
|
||||
python -m pip install --quiet pytest
|
||||
python3 -m pytest tests/test_lint_required_context_exists_in_bp.py -v
|
||||
@@ -69,7 +69,6 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: meta-lint advisory; CI / all-required is the required aggregate.
|
||||
lint:
|
||||
name: lint-required-no-paths
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
name: lint-required-workflows-docker-host-pinned
|
||||
|
||||
# Fail-closed lint that catches workflows touching docker.sock without
|
||||
# pinning `runs-on:` to a Linux-only label.
|
||||
#
|
||||
# Class defect (internal#512 + mc#1529 + today's oc#81/82/83 + autogen#8):
|
||||
# the `ubuntu-latest` label is advertised by BOTH the Linux operator-host
|
||||
# runners (molecule-runner-*) AND the Windows act_runner v1.0.3 on
|
||||
# hongming-pc-runner-*. Job placement is non-deterministic. When a docker-
|
||||
# bound job lands on a Windows runner, `docker run`/`docker login`/
|
||||
# `docker compose` fail with platform-specific errors ("protocol not
|
||||
# available", "cannot exec", etc.) — placement-dependent, not transient.
|
||||
#
|
||||
# This lint enforces the convention: any workflow whose YAML body
|
||||
# contains a docker exec (`docker run|build|buildx|compose|pull|push|
|
||||
# exec|tag|login|cp|inspect|ps` OR `docker/build-push-action|docker/
|
||||
# login-action|docker/setup-buildx`) MUST pin every job's `runs-on:` to
|
||||
# one of:
|
||||
# - docker-host (general docker.sock work — molecule-runner-*)
|
||||
# - publish (image build/push — molecule-runner-publish-*)
|
||||
#
|
||||
# Comments and heredoc/markdown bodies that merely MENTION docker are
|
||||
# excluded by the detection rule (see scan.py below).
|
||||
#
|
||||
# Per `feedback_never_skip_ci`: this is fail-closed (exit 1 on miss).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
lint-docker-host-pin:
|
||||
name: Lint docker-host pin on docker-touching workflows
|
||||
runs-on: docker-host
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Scan workflows for docker-bound jobs missing docker-host/publish pin
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Docker-step detection: real exec, not just word-mention in comments.
|
||||
# We strip comment-only lines, then look for the docker subcommand
|
||||
# tokens at word-boundary, OR uses: docker/* actions.
|
||||
DOCKER_EXEC = re.compile(
|
||||
r'(?<!\w)docker\s+(run|build|buildx|compose|pull|push|exec|tag|login|cp|inspect|ps)\b'
|
||||
)
|
||||
DOCKER_ACTION = re.compile(
|
||||
r'uses:\s*docker/(build-push-action|login-action|setup-buildx-action|setup-qemu-action)'
|
||||
)
|
||||
# Detect a job header line like ` myjob:` (2-space indent) AND its runs-on.
|
||||
JOB_HEADER = re.compile(r'^( {2})([a-zA-Z0-9_-]+):\s*$')
|
||||
RUNS_ON = re.compile(r'^( {4})runs-on:\s*(.+?)\s*$')
|
||||
|
||||
ALLOWED_LABELS = {'docker-host', 'publish'}
|
||||
|
||||
fails = []
|
||||
warnings = []
|
||||
|
||||
# Gitea is SSOT for molecule-core CI per task #347 / memory
|
||||
# reference_molecule_core_actions_gitea_only. The legacy
|
||||
# .github/workflows/ tree was deleted in SSOT-Instance-4 (#331).
|
||||
roots = []
|
||||
for root in ('.gitea/workflows',):
|
||||
if os.path.isdir(root):
|
||||
roots.append(root)
|
||||
|
||||
for root in roots:
|
||||
for fn in sorted(os.listdir(root)):
|
||||
if not (fn.endswith('.yml') or fn.endswith('.yaml')):
|
||||
continue
|
||||
path = os.path.join(root, fn)
|
||||
with open(path) as f:
|
||||
raw_lines = f.readlines()
|
||||
|
||||
# Parse job headers + their runs-on. Simple line scan; relies on
|
||||
# 2-space job indent + 4-space runs-on indent under `jobs:`.
|
||||
jobs = []
|
||||
current = None
|
||||
in_jobs = False
|
||||
for i, line in enumerate(raw_lines, 1):
|
||||
if re.match(r'^jobs:\s*$', line):
|
||||
in_jobs = True
|
||||
continue
|
||||
if not in_jobs:
|
||||
continue
|
||||
mh = JOB_HEADER.match(line)
|
||||
if mh:
|
||||
if current:
|
||||
current['end'] = i - 1
|
||||
jobs.append(current)
|
||||
current = {'name': mh.group(2), 'line': i, 'end': len(raw_lines), 'runs_on': None}
|
||||
continue
|
||||
mr = RUNS_ON.match(line)
|
||||
if mr and current and current['runs_on'] is None:
|
||||
current['runs_on'] = mr.group(2).strip()
|
||||
if current:
|
||||
jobs.append(current)
|
||||
|
||||
for j in jobs:
|
||||
# Strip pure-comment lines for docker-exec detection so
|
||||
# documentation comments don't trigger the lint. Scan the
|
||||
# current job body only: a workflow may contain one
|
||||
# docker-bound job and several harmless metadata jobs.
|
||||
job_lines = raw_lines[j['line'] - 1:j['end']]
|
||||
scan_text = ''.join(
|
||||
l for l in job_lines
|
||||
if not re.match(r'^\s*#', l)
|
||||
)
|
||||
has_docker = bool(DOCKER_EXEC.search(scan_text)) or bool(DOCKER_ACTION.search(scan_text))
|
||||
if not has_docker:
|
||||
continue
|
||||
ro = j['runs_on']
|
||||
if ro is None:
|
||||
# Reusable workflow caller (`uses:` instead of `runs-on:`) —
|
||||
# skip; rule enforced in the called workflow.
|
||||
continue
|
||||
# Strip surrounding [ ] and quotes.
|
||||
ro_norm = ro.strip('[]').strip().strip('"\'')
|
||||
# Multi-label "[a, b]" — split.
|
||||
labels = [t.strip().strip('"\'') for t in ro_norm.split(',') if t.strip()]
|
||||
if any(lbl in ALLOWED_LABELS for lbl in labels):
|
||||
continue
|
||||
# Allow caller-supplied label expressions; spell the
|
||||
# marker indirectly so Gitea's expression parser does
|
||||
# not try to parse this Python heredoc.
|
||||
expression_marker = '$' + '{{'
|
||||
if any(expression_marker in lbl for lbl in labels):
|
||||
continue
|
||||
fails.append(
|
||||
f"{path}:{j['line']}: job `{j['name']}` uses docker but runs-on={ro!r} "
|
||||
f"(must be one of {sorted(ALLOWED_LABELS)})"
|
||||
)
|
||||
|
||||
if fails:
|
||||
print("FAIL: docker-bound jobs missing docker-host/publish pin:")
|
||||
for f in fails:
|
||||
print(f" - {f}")
|
||||
print()
|
||||
print("Why this rule exists (internal#512 + mc#1529):")
|
||||
print(" Bare `ubuntu-latest` is advertised by BOTH Linux operator-host")
|
||||
print(" runners AND Windows hongming-pc-runner-* (act_runner v1.0.3).")
|
||||
print(" Docker-bound jobs that land on Windows fail non-deterministically.")
|
||||
print(" Pin to `docker-host` (general) or `publish` (image build/push).")
|
||||
sys.exit(1)
|
||||
|
||||
print("OK: all docker-bound jobs are pinned to docker-host or publish.")
|
||||
PY
|
||||
@@ -1,117 +0,0 @@
|
||||
name: Lint shellcheck (arm64 pilot)
|
||||
|
||||
# Mac-CI dual-track pilot (#233). ADDITIVE / NOT REQUIRED.
|
||||
#
|
||||
# Validates the arm64 self-hosted lane (no docker.sock, no privileged
|
||||
# ops) before any required gate moves onto it.
|
||||
#
|
||||
# Runner label mapping (2026-05-22 fix): the actual Mac mini runner
|
||||
# registered in this Gitea ships labels
|
||||
# ["self-hosted","macos-self-hosted-arm64","arm64-darwin"]
|
||||
# — no plain `arm64`. The earlier `runs-on: [self-hosted, arm64]`
|
||||
# could not match any registered runner so every fire of this workflow
|
||||
# was assigned task_id=0 / runner_id=NULL → Gitea cancelled it. The
|
||||
# rows showed up as Cancelled in the action status feed (not Failed)
|
||||
# but the lane never actually ran. Workflow now selects on
|
||||
# `arm64-darwin` which is the canonical Mac-arm64 label per the
|
||||
# Mac mini's registration (per internal#494 capability-honest labels).
|
||||
#
|
||||
# If we later want to add a Linux-arm64 runner to the same lane, add
|
||||
# both labels to that runner's registration AND broaden the selector
|
||||
# here — don't rename `arm64-darwin` (it's Mac-specific by design and
|
||||
# `feedback_pc2_runner_labels_must_stay_narrow` rule applies).
|
||||
#
|
||||
# Pairs with internal#543 (RFC: Mac arm64 multi-arch runner-base) and
|
||||
# internal#494 (multi-arch runner-base capability-honest labels).
|
||||
# No paths: filter on purpose (feedback_path_filtered_workflow_cant_be_required).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- staging
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
shellcheck-arm64:
|
||||
name: shellcheck-arm64 (pilot)
|
||||
runs-on: [self-hosted, arm64-darwin]
|
||||
# NOT a required check; safe to sit pending until Mac runner is up.
|
||||
# If the Mac runner has trouble pulling actions/checkout we fall
|
||||
# back to a plain git clone (see step 'fallback clone').
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
set -eu
|
||||
echo "arch=$(uname -m)"
|
||||
echo "kernel=$(uname -sr)"
|
||||
echo "shell=$BASH_VERSION"
|
||||
# Sanity: must actually be arm64. If amd64 sneaks in here,
|
||||
# fail fast — that means the label routing is wrong.
|
||||
case "$(uname -m)" in
|
||||
aarch64|arm64) echo "arm64 confirmed" ;;
|
||||
*) echo "ERROR: expected arm64, got $(uname -m)"; exit 1 ;;
|
||||
esac
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Install shellcheck (arm64)
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -eu
|
||||
if command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "shellcheck already present: $(shellcheck --version | head -1)"
|
||||
else
|
||||
# Prefer apt if the runner base ships it; else download arm64 binary.
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y --no-install-recommends shellcheck
|
||||
else
|
||||
SC_VER=v0.10.0
|
||||
curl -fsSL "https://github.com/koalaman/shellcheck/releases/download/${SC_VER}/shellcheck-${SC_VER}.linux.aarch64.tar.xz" \
|
||||
| tar -xJf - --strip-components=1
|
||||
sudo mv shellcheck /usr/local/bin/
|
||||
fi
|
||||
fi
|
||||
shellcheck --version | head -2
|
||||
|
||||
- name: Run shellcheck on .gitea/scripts/*.sh
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -eu
|
||||
# Only the scripts we control under .gitea/scripts. Pilot
|
||||
# scope is intentionally narrow — broaden in a follow-up
|
||||
# once the lane is proven.
|
||||
if ! command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "WARN: shellcheck binary not found — skipping (pilot mode)"
|
||||
exit 0
|
||||
fi
|
||||
# NOTE: macOS ships Bash 3.2 (Apple license), no `mapfile`
|
||||
# (Bash 4+ builtin). Mac mini runner empirically failed at
|
||||
# `mapfile: command not found` (run 79275 / task 145654).
|
||||
# Use the portable `while read` pattern instead — works on
|
||||
# both Bash 3.2 (macOS) and Bash 4+ (Linux).
|
||||
TARGETS=()
|
||||
while IFS= read -r f; do
|
||||
TARGETS+=("$f")
|
||||
done < <(find .gitea/scripts -maxdepth 2 -type f -name '*.sh' | sort)
|
||||
if [ "${#TARGETS[@]}" -eq 0 ]; then
|
||||
echo "No .sh files found under .gitea/scripts — nothing to check"
|
||||
exit 0
|
||||
fi
|
||||
echo "Checking ${#TARGETS[@]} file(s):"
|
||||
printf ' %s\n' "${TARGETS[@]}"
|
||||
# SC1091 = couldn't follow non-constant source; expected for
|
||||
# CI-time analysis without the full runtime layout.
|
||||
shellcheck --severity=error --exclude=SC1091 "${TARGETS[@]}"
|
||||
@@ -55,7 +55,6 @@ jobs:
|
||||
# Phase 3 (RFC #219 §1): surface broken shapes without blocking PRs.
|
||||
# Follow-up PR flips this off after the 4 existing-on-main rule-2
|
||||
# (workflow_run) violations are migrated to a supported trigger.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -9,12 +9,18 @@ name: publish-canvas-image
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - Retargeted the image push from GHCR to ECR. GHCR was retired during
|
||||
# the 2026-05-06 Gitea migration, and Gitea's GITHUB_TOKEN cannot
|
||||
# authenticate to ghcr.io.
|
||||
# - **Open question for review**: this workflow pushes the canvas
|
||||
# image to `ghcr.io`. GHCR was retired during the 2026-05-06
|
||||
# Gitea migration in favor of ECR (per staging-verify.yml header
|
||||
# notes). The image may not be consumable post-migration. Two
|
||||
# options for follow-up: (a) retarget to
|
||||
# `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas`,
|
||||
# or (b) retire this workflow entirely and route canvas deploys
|
||||
# via the operator-host build path. tier:low + continue-on-error
|
||||
# means failed pushes do not block PRs.
|
||||
#
|
||||
|
||||
# Builds and pushes the canvas Docker image to ECR whenever a commit lands
|
||||
# Builds and pushes the canvas Docker image to GHCR whenever a commit lands
|
||||
# on main that touches canvas code. Previously canvas changes were visible in
|
||||
# CI (npm run build passed) but the live container was never updated —
|
||||
# operators had to manually run `docker compose build canvas` each time.
|
||||
@@ -39,82 +45,38 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
packages: write # required to push to ghcr.io/${{ github.repository_owner }}/*
|
||||
|
||||
env:
|
||||
# SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
|
||||
# sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
|
||||
# bootstrap fallback. When the org var is set, the fallback becomes dead code and
|
||||
# switching accounts/regions is a one-line change at the org level (instead of
|
||||
# touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
|
||||
# use below in this repo's staging-verify.yml.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/canvas
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/canvas
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: post-merge image publication side effect; CI / all-required gates source changes.
|
||||
build-and-push:
|
||||
name: Build & push canvas image
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399). Ship
|
||||
# path (on: push:main, canvas/**) — reserved capacity so a merged
|
||||
# canvas fix's image build never FIFO-queues behind PR required-CI.
|
||||
# The `publish` label resolves ONLY to the molecule-runner-publish-*
|
||||
# sub-pool (config.publish.yaml). HARD DEPENDENCY: this MUST land
|
||||
# AFTER the publish-lane runners are registered/advertising `publish`
|
||||
# — the earlier #599 `docker` label attempt queued indefinitely with
|
||||
# zero eligible runners precisely because the label was targeted
|
||||
# before any runner advertised it (see #576). The lane is registered
|
||||
# in this rollout (internal#462) so the precondition holds.
|
||||
runs-on: publish
|
||||
# REVERTED (infra/revert-docker-runner-label): `runs-on: ubuntu-latest` restored.
|
||||
# The `docker` label is not registered on any act_runner. `runs-on: [ubuntu-latest, docker]`
|
||||
# causes jobs to queue indefinitely with zero eligible runners — strictly worse than the
|
||||
# pre-#599 coin-flip (50% success rate). Once the `docker` label is registered on
|
||||
# ≥2 runners, re-apply the fix from #599 (infra/docker-runner-label).
|
||||
# See issue #576 + infra-lead pulse ~00:30Z.
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Keep Docker auth/buildx state inside the job temp dir. Publish
|
||||
# runners can inherit a HOME/DOCKER_CONFIG path that is host-owned
|
||||
# and not writable from the job container; docker login otherwise
|
||||
# fails before the image build starts.
|
||||
- name: Prepare writable Docker config
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export DOCKER_CONFIG="$RUNNER_TEMP/docker-config"
|
||||
mkdir -p "$DOCKER_CONFIG/buildx/certs"
|
||||
echo "DOCKER_CONFIG=$DOCKER_CONFIG" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Log in to ECR
|
||||
env:
|
||||
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ECR_REGISTRY="${IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Ensure ECR repository exists
|
||||
env:
|
||||
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
repo_path="${IMAGE_NAME#*/}"
|
||||
if ! aws ecr describe-repositories --repository-names "${repo_path}" --region us-east-2 >/dev/null 2>&1; then
|
||||
aws ecr create-repository \
|
||||
--repository-name "${repo_path}" \
|
||||
--image-scanning-configuration scanOnPush=true \
|
||||
--region us-east-2 >/dev/null
|
||||
fi
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing to the build step
|
||||
@@ -124,14 +86,12 @@ jobs:
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
echo "Runner: ${HOSTNAME:-unknown}"
|
||||
docker_info="$(docker info 2>&1)" || {
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Runner: ${HOSTNAME:-unknown}"
|
||||
printf '%s\n' "${docker_info}"
|
||||
echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "${docker_info}" | sed -n '1,5p'
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
|
||||
@@ -165,7 +125,7 @@ jobs:
|
||||
echo "platform_url=${PLATFORM_URL}" >> "$GITHUB_OUTPUT"
|
||||
echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build & push canvas image to ECR
|
||||
- name: Build & push canvas image to GHCR
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ./canvas
|
||||
@@ -178,10 +138,9 @@ jobs:
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:sha-${{ steps.tags.outputs.sha }}
|
||||
# Gitea artifact-cache reachability is best-effort on the operator
|
||||
# runner network. Do not let cache export fail an image that already
|
||||
# built and pushed successfully.
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
labels: |
|
||||
org.opencontainers.image.source=https://git.moleculesai.app/${{ github.repository }}
|
||||
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
org.opencontainers.image.description=Molecule AI canvas (Next.js 15 + React Flow)
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
name: publish-runtime-autobump
|
||||
|
||||
# Auto-bump-on-workspace-edit half of the publish pipeline.
|
||||
#
|
||||
# Why this file exists (issue #351):
|
||||
# Gitea Actions does not correctly disambiguate `paths:` from `tags:`
|
||||
# when both are bundled under a single `on.push` key. The result is
|
||||
# that tag pushes get filtered out and `publish-runtime.yml` never
|
||||
# fires — `action_run` rows: 0. This was unnoticed pre-2026-05-11
|
||||
# because PYPI_TOKEN was absent (publishes would have failed anyway).
|
||||
#
|
||||
# Split design:
|
||||
# - publish-runtime.yml : on.push.tags only (the publisher)
|
||||
# - publish-runtime-autobump.yml: on.push.branches+paths (this file — the version-bumper)
|
||||
#
|
||||
# This file computes the next version from PyPI's latest, pushes a
|
||||
# `runtime-v$VERSION` tag, and exits. The tag push then triggers
|
||||
# publish-runtime.yml via its tags-only trigger.
|
||||
#
|
||||
# Concurrency: shares the `publish-runtime` group with publish-runtime.yml
|
||||
# so concurrent workspace pushes serialize at the bump step. Without
|
||||
# this, two pushes minutes apart could both read PyPI latest=0.1.129
|
||||
# and try to tag 0.1.130 simultaneously, only one of which would land.
|
||||
|
||||
on:
|
||||
# Run on PR pushes to post a success status so Gitea can merge the PR.
|
||||
# All steps use continue-on-error: true so operational failures
|
||||
# (PyPI unreachable, DISPATCH_TOKEN missing) do not block merge.
|
||||
pull_request:
|
||||
paths:
|
||||
- "workspace/**"
|
||||
# Bump-and-tag on main/staging push (the actual operational trigger).
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- staging
|
||||
paths:
|
||||
- "workspace/**"
|
||||
# Manual dispatch — useful when Gitea Actions API (/actions/*) is
|
||||
# unreachable (e.g. act_runner 404 on Gitea 1.22.6) and we cannot
|
||||
# re-trigger via curl.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write # required to push tags back
|
||||
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# PR-validation path: always succeeds so Gitea can merge workflow-only PRs.
|
||||
# Operational failures (PyPI unreachable, missing DISPATCH_TOKEN) are
|
||||
# surfaced via continue-on-error: true rather than blocking the merge.
|
||||
# The actual bump work happens on the main/staging push after merge.
|
||||
pr-validate:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # do not block PR merge on operational failures
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Validate PyPI connectivity (best-effort)
|
||||
run: |
|
||||
set -eu
|
||||
echo "=== Checking PyPI accessibility ==="
|
||||
LATEST=$(curl -fsS --retry 3 --max-time 10 \
|
||||
https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])" \
|
||||
|| echo "PyPI unreachable (non-blocking for PR validation)")
|
||||
echo "Latest: ${LATEST:-unknown}"
|
||||
|
||||
# Actual bump-and-tag: runs on main/staging pushes, posts real success/failure.
|
||||
# No continue-on-error — operational failures here trip the main-red
|
||||
# watchdog, which is the desired signal for infrastructure degradation.
|
||||
bump-and-tag:
|
||||
runs-on: ubuntu-latest
|
||||
# Only fire on push events (main/staging after PR merge). Pull_request
|
||||
# events are handled by pr-validate above; we do NOT bump on every
|
||||
# push-synchronize because that would race with the PR head.
|
||||
#
|
||||
# NOTE: the prior condition `github.event.pull_request.base.ref == ''`
|
||||
# was broken — on a PR-merge push in Gitea Actions, the pull_request
|
||||
# context is still attached (base.ref='main'), so the condition always
|
||||
# evaluated to false and bump-and-tag was permanently skipped.
|
||||
if: github.event_name == 'push'
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Fetch tags for collision check
|
||||
run: git fetch origin --tags --depth=1
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Compute next version from PyPI latest
|
||||
id: bump
|
||||
run: |
|
||||
set -eu
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "PyPI latest=$LATEST -> next=$VERSION"
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
|
||||
echo "::error::computed version $VERSION does not match PEP 440 X.Y.Z"
|
||||
exit 1
|
||||
fi
|
||||
if git tag --list | grep -qx "runtime-v$VERSION"; then
|
||||
echo "::error::tag runtime-v$VERSION already exists in this repo. Manual intervention required (PyPI and Gitea tag history are out of sync)."
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Push runtime-v$VERSION tag
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
VERSION: ${{ steps.bump.outputs.version }}
|
||||
GITEA_URL: https://git.moleculesai.app
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
echo "::error::DISPATCH_TOKEN secret is not set — needed to push the tag back to molecule-core."
|
||||
exit 1
|
||||
fi
|
||||
git config user.name "publish-runtime autobump"
|
||||
git config user.email "publish-runtime@moleculesai.app"
|
||||
git tag -a "runtime-v$VERSION" \
|
||||
-m "Auto-bump on workspace/** edit on $GITHUB_REF" \
|
||||
-m "Triggered by: $GITHUB_REF @ $GITHUB_SHA" \
|
||||
-m "publish-runtime.yml will pick up this tag and upload to PyPI"
|
||||
# Push via DISPATCH_TOKEN (a Gitea PAT). Using the bot identity
|
||||
# ensures the resulting tag-push event is dispatched to
|
||||
# publish-runtime.yml; act_runner's default GITHUB_TOKEN cannot
|
||||
# trigger downstream workflows.
|
||||
git remote set-url origin "${GITEA_URL#https://}"
|
||||
git remote set-url origin "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/molecule-ai/molecule-core.git"
|
||||
git push origin "runtime-v$VERSION"
|
||||
echo "✓ pushed runtime-v$VERSION — publish-runtime.yml should fire next"
|
||||
@@ -0,0 +1,339 @@
|
||||
name: publish-runtime
|
||||
|
||||
# Gitea Actions port of .github/workflows/publish-runtime.yml.
|
||||
#
|
||||
# Ported 2026-05-10 (issue #206). Key differences from the GitHub version:
|
||||
# - Gitea Actions reads .gitea/workflows/, not .github/workflows/
|
||||
# - Dropped `environment: pypi-publish` — Gitea Actions does not support
|
||||
# named environments or OIDC trusted publishers
|
||||
# - Replaced `pypa/gh-action-pypi-publish@release/v1` (OIDC) with
|
||||
# `twine upload` using PYPI_TOKEN secret — same mechanism as a local
|
||||
# `python -m twine upload` with a PyPI token
|
||||
# - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/tags/}`
|
||||
# — Gitea Actions exposes github.ref (the full ref) but not ref_name
|
||||
# - Dropped `merge_group` trigger (Gitea has no merge queue)
|
||||
#
|
||||
# 2026-05-10 (issue #348): originally restored `staging`/`main` branch +
|
||||
# `workspace/**` path-filter trigger in PR #349.
|
||||
#
|
||||
# 2026-05-11 (issue #351): REVERTED the branches+paths trigger from THIS
|
||||
# file. Bundling `paths` with `tags` under a single `on.push` key caused
|
||||
# Gitea Actions to never dispatch the workflow for tag-push events (0
|
||||
# runs in `action_run` for workflow_id='publish-runtime.yml' since the
|
||||
# port, including the runtime-v1.0.0 tag — which is why PyPI is still at
|
||||
# 0.1.129 despite a v1.0.0 Gitea tag existing).
|
||||
#
|
||||
# The auto-bump-on-workspace-edit trigger now lives in
|
||||
# `.gitea/workflows/publish-runtime-autobump.yml`. That file computes the
|
||||
# next version from PyPI's latest and pushes a `runtime-v$VERSION` tag,
|
||||
# which THIS file then picks up via the tags-only trigger below.
|
||||
#
|
||||
# This decoupling means Gitea's path-vs-tag evaluator never has to
|
||||
# disambiguate — each file has a single unambiguous trigger shape.
|
||||
#
|
||||
# PyPI publishing: requires PYPI_TOKEN repository secret (or org-level secret).
|
||||
# Set via: repo Settings → Actions → Variables and Secrets → New Secret.
|
||||
# The token should be a PyPI API token scoped to molecule-ai-workspace-runtime.
|
||||
#
|
||||
# The DISPATCH_TOKEN cascade (git push to template repos) is unchanged —
|
||||
# it uses the Gitea API directly and was already Gitea-compatible.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "runtime-v*"
|
||||
workflow_dispatch:
|
||||
# 2026-05-11 (root cause of #351 / 0 runs ever):
|
||||
# Gitea 1.22.6's workflow parser rejects `workflow_dispatch.inputs.version`
|
||||
# with "unknown on type" — it mis-treats the inputs sub-keys as top-level
|
||||
# `on:` event types. Log line:
|
||||
# actions/workflows.go:DetectWorkflows() [W] ignore invalid workflow
|
||||
# "publish-runtime.yml": unknown on type: map["version": {...}]
|
||||
# That `[W] ignore invalid workflow` is silent UX — the workflow never
|
||||
# registers, so it never fires for ANY event (push.tags included).
|
||||
# Removing the inputs block restores parsing. Manual dispatch from the
|
||||
# Gitea UI now triggers the PyPI auto-bump fallback in `Derive version`
|
||||
# below (no `inputs.version` to read).
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Serialize publishes so two concurrent tag pushes don't both compute
|
||||
# "latest+1" and race on PyPI upload. The second one waits.
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
|
||||
- name: Derive version (tag or PyPI auto-bump)
|
||||
id: version
|
||||
run: |
|
||||
if echo "$GITHUB_REF" | grep -q "^refs/tags/runtime-v"; then
|
||||
# Tag is `runtime-vX.Y.Z` — strip the prefix.
|
||||
VERSION="${GITHUB_REF#refs/tags/runtime-v}"
|
||||
else
|
||||
# workflow_dispatch path (no inputs supported on Gitea 1.22.6) or
|
||||
# any other non-tag trigger: derive from PyPI latest + patch bump.
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
|
||||
fi
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
|
||||
echo "::error::version $VERSION does not match PEP 440"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Publishing molecule-ai-workspace-runtime $VERSION"
|
||||
|
||||
- name: Install build tooling
|
||||
run: pip install build twine
|
||||
|
||||
- name: Build package from workspace/
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "${{ steps.version.outputs.version }}" \
|
||||
--out "${{ runner.temp }}/runtime-build"
|
||||
|
||||
- name: Build wheel + sdist
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: python -m build
|
||||
|
||||
- name: Capture wheel SHA256 for cascade content-verification
|
||||
id: wheel_hash
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
set -eu
|
||||
WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
|
||||
if [ -z "$WHEEL" ]; then
|
||||
echo "::error::No .whl in dist/ — \`python -m build\` must have failed silently"
|
||||
exit 1
|
||||
fi
|
||||
HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
|
||||
echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
|
||||
echo "Local wheel SHA256 (pre-upload): ${HASH}"
|
||||
echo "Wheel filename: $(basename "$WHEEL")"
|
||||
|
||||
- name: Verify package contents (sanity)
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
python -m twine check dist/*
|
||||
python -m venv /tmp/smoke
|
||||
/tmp/smoke/bin/pip install --quiet dist/*.whl
|
||||
/tmp/smoke/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
|
||||
- name: Publish to PyPI
|
||||
# working-directory matches the preceding Build/Verify steps. Without
|
||||
# this, twine runs from the default workspace checkout dir where
|
||||
# `dist/` doesn't exist and fails with:
|
||||
# ERROR InvalidDistribution: Cannot find file (or expand pattern): 'dist/*'
|
||||
# Caught on the first-ever successful dispatch of this workflow
|
||||
# (run 5097, 2026-05-11 02:08Z) — every other step in the publish
|
||||
# job already had this working-directory; Publish was missing it.
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
env:
|
||||
# PYPI_TOKEN: repository secret scoped to molecule-ai-workspace-runtime.
|
||||
# Set via: Settings → Actions → Variables and Secrets → New Secret.
|
||||
# Format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$PYPI_TOKEN" ]; then
|
||||
echo "::error::PYPI_TOKEN secret is not set — set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
echo "::error::Required format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
|
||||
exit 1
|
||||
fi
|
||||
python -m twine upload \
|
||||
--repository pypi \
|
||||
--username __token__ \
|
||||
--password "$PYPI_TOKEN" \
|
||||
dist/*
|
||||
|
||||
cascade:
|
||||
needs: publish
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Wait for PyPI to propagate the new version
|
||||
env:
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
python -m venv /tmp/propagation-probe
|
||||
PROBE=/tmp/propagation-probe/bin
|
||||
$PROBE/pip install --upgrade --quiet pip
|
||||
for i in $(seq 1 30); do
|
||||
if $PROBE/pip install \
|
||||
--quiet \
|
||||
--no-cache-dir \
|
||||
--force-reinstall \
|
||||
--no-deps \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1; then
|
||||
INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
|
||||
| awk -F': ' '/^Version:/{print $2}')
|
||||
if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
|
||||
echo "✓ PyPI resolved $RUNTIME_VERSION (install check)"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "::error::pip install --no-cache-dir molecule-ai-workspace-runtime==${RUNTIME_VERSION} never resolved within ~5 min."
|
||||
echo "::error::Refusing to fan out cascade against a potentially stale PyPI index."
|
||||
exit 1
|
||||
fi
|
||||
echo " [$i/30] waiting for PyPI to propagate ${RUNTIME_VERSION}..."
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Stage (b): download wheel + SHA256 compare against what we built.
|
||||
# Catches Fastly stale-content serving old bytes under a new version URL.
|
||||
#
|
||||
# Caught run 5196 (first-ever successful publish, 2026-05-11): the
|
||||
# previous one-liner `HASH=$(pip download ... && sha256sum ...)`
|
||||
# captured pip's stdout (`Collecting molecule-ai-workspace-runtime
|
||||
# ==X.Y.Z`) into HASH, then the SHA comparison failed against the
|
||||
# leaked `Collecting...` string. `2>/dev/null` silences stderr but
|
||||
# NOT stdout; pip writes its progress to stdout by default.
|
||||
# Fix: split into two steps, silence pip's stdout explicitly, capture
|
||||
# only sha256sum's output into HASH.
|
||||
python -m pip download \
|
||||
--no-deps \
|
||||
--no-cache-dir \
|
||||
--dest /tmp/wheel-probe \
|
||||
--quiet \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1
|
||||
HASH=$(sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
|
||||
if [ "$HASH" != "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::PyPI propagated $RUNTIME_VERSION but wheel content SHA256 mismatch."
|
||||
echo "::error::Expected: $EXPECTED_SHA256"
|
||||
echo "::error::Got: $HASH"
|
||||
echo "::error::Fastly may be serving stale content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ PyPI CDN verified (SHA256 match)"
|
||||
|
||||
- name: Fan out via push to .runtime-version
|
||||
env:
|
||||
# Gitea PAT with write:repository scope on the 8 cascade-active
|
||||
# template repos. Used for git push to each template repo's main
|
||||
# branch, which trips their `on: push: branches: [main]` trigger
|
||||
# on publish-image.yml.
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
run: |
|
||||
set +e # don't abort on a single repo failure — collect them all
|
||||
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::DISPATCH_TOKEN secret not set — skipping cascade."
|
||||
echo "::warning::set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::DISPATCH_TOKEN secret missing — cascade cannot fan out."
|
||||
echo "::error::PyPI was published, but the 8 template repos will NOT pick up the new version."
|
||||
exit 1
|
||||
fi
|
||||
VERSION="$RUNTIME_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::publish job did not expose a version output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GITEA_URL="${GITEA_URL:-https://git.moleculesai.app}"
|
||||
TEMPLATES="claude-code hermes openclaw codex langgraph crewai autogen deepagents gemini-cli"
|
||||
FAILED=""
|
||||
SKIPPED=""
|
||||
|
||||
git config --global user.name "publish-runtime cascade"
|
||||
git config --global user.email "publish-runtime@moleculesai.app"
|
||||
|
||||
WORKDIR="$(mktemp -d)"
|
||||
for tpl in $TEMPLATES; do
|
||||
REPO="molecule-ai/molecule-ai-workspace-template-$tpl"
|
||||
CLONE="$WORKDIR/$tpl"
|
||||
|
||||
HTTP=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $DISPATCH_TOKEN" \
|
||||
"$GITEA_URL/api/v1/repos/$REPO/contents/.github/workflows/publish-image.yml")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "↷ $tpl has no publish-image.yml — soft-skip"
|
||||
SKIPPED="$SKIPPED $tpl"
|
||||
continue
|
||||
fi
|
||||
|
||||
attempt=0
|
||||
success=false
|
||||
while [ $attempt -lt 3 ]; do
|
||||
attempt=$((attempt + 1))
|
||||
rm -rf "$CLONE"
|
||||
if ! git clone --depth=1 \
|
||||
"https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/$REPO.git" \
|
||||
"$CLONE" >/tmp/clone.log 2>&1; then
|
||||
echo "::warning::clone $tpl attempt $attempt failed: $(tail -n3 /tmp/clone.log)"
|
||||
sleep 2
|
||||
continue
|
||||
fi
|
||||
|
||||
cd "$CLONE"
|
||||
echo "$VERSION" > .runtime-version
|
||||
|
||||
if git diff --quiet -- .runtime-version; then
|
||||
echo "✓ $tpl already at $VERSION — no commit needed"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
git add .runtime-version
|
||||
git commit -m "chore: pin runtime to $VERSION (publish-runtime cascade)" \
|
||||
-m "Co-Authored-By: publish-runtime cascade <publish-runtime@moleculesai.app>" \
|
||||
>/dev/null
|
||||
|
||||
if git push origin HEAD:main >/tmp/push.log 2>&1; then
|
||||
echo "✓ $tpl pushed $VERSION on attempt $attempt"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
echo "::warning::push $tpl attempt $attempt failed, pull-rebasing"
|
||||
git pull --rebase origin main >/tmp/rebase.log 2>&1 || true
|
||||
cd - >/dev/null
|
||||
done
|
||||
|
||||
if [ "$success" != "true" ]; then
|
||||
FAILED="$FAILED $tpl"
|
||||
fi
|
||||
done
|
||||
rm -rf "$WORKDIR"
|
||||
|
||||
if [ -n "$FAILED" ]; then
|
||||
echo "::error::Cascade incomplete after 3 retries each. Failed:$FAILED"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$SKIPPED" ]; then
|
||||
echo "Cascade complete: pinned $VERSION. Soft-skipped (no publish-image.yml):$SKIPPED"
|
||||
else
|
||||
echo "Cascade complete: $VERSION pinned across all manifest workspace_templates."
|
||||
fi
|
||||
@@ -18,19 +18,8 @@ name: publish-workspace-server-image
|
||||
# :staging-<sha> — per-commit digest, stable for canary verify
|
||||
# :staging-latest — tracks most recent build on this branch
|
||||
#
|
||||
# Production auto-deploy:
|
||||
# After both platform and tenant images are pushed, deploy-production waits
|
||||
# for strict required push contexts on the same SHA to go green, then
|
||||
# calls the production CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha>. Set repo variable or secret PROD_AUTO_DEPLOY_DISABLED=true
|
||||
# to stop production rollout while keeping image publishing enabled.
|
||||
#
|
||||
# Primary ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
|
||||
# Optional staging tenant mirror target:
|
||||
# 004947743811.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
|
||||
# Required secrets: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AUTO_SYNC_TOKEN
|
||||
# Staging ECR grants the primary SSOT-managed publisher principal repository
|
||||
# policy access, so no persistent staging AWS access keys are required.
|
||||
#
|
||||
# mc#711: Docker daemon not accessible on ubuntu-latest runner (molecule-canonical-1
|
||||
# shows client-only in `docker info` — daemon not running). DinD mount is present but
|
||||
@@ -41,68 +30,53 @@ name: publish-workspace-server-image
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'manifest.json'
|
||||
- 'scripts/**'
|
||||
- '.gitea/workflows/publish-workspace-server-image.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
# No `concurrency:` block here. Gitea 1.22.6 can cancel queued runs despite
|
||||
# `cancel-in-progress: false`; that is not acceptable for a workflow with a
|
||||
# production deploy job. Per-SHA image tags are immutable, and staging-latest is
|
||||
# best-effort last-writer-wins metadata.
|
||||
# Serialize per-branch so two rapid main pushes don't race the same
|
||||
# :staging-latest tag retag. Allow parallel runs as they produce
|
||||
# different :staging-<sha> tags and last-write-wins on :staging-latest.
|
||||
#
|
||||
# 2026-05-20 retrigger: run #86994 on mc#1589 merge sha 0f0f1ba2 failed at
|
||||
# setup-buildx-action with EACCES on PC2 WSL publish runner — the runner's
|
||||
# DOCKER_CONFIG=/home/hongming/.docker-ecr/ dir didn't have a buildx/certs
|
||||
# subdir writable by the container's UID 1001. Hot-patched the dir perms;
|
||||
# this chore push retriggers the workflow. Proper fix (per-runner
|
||||
# DOCKER_CONFIG owned by 1001, internal#597 --env HOME=/home/runner pattern)
|
||||
# is tracked as a CI-hygiene follow-up — not in scope here.
|
||||
# cancel-in-progress: false → in-flight builds finish; the next push's
|
||||
# build queues. This avoids a partially-pushed image.
|
||||
concurrency:
|
||||
group: publish-workspace-server-image-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
# SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
|
||||
# sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
|
||||
# bootstrap fallback. When the org var is set, the fallback becomes dead code and
|
||||
# switching accounts/regions is a one-line change at the org level (instead of
|
||||
# touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
|
||||
# use below in this repo's staging-verify.yml.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
STAGING_TENANT_IMAGE_NAME: ${{ vars.STAGING_ECR_REGISTRY || '004947743811.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399). This
|
||||
# is a post-merge ship job (on: push:main) — it must NOT FIFO-compete
|
||||
# with PR required-CI on the shared pool (PR#1350's prod image build
|
||||
# was delayed ~25min this way). The `publish` label resolves ONLY to
|
||||
# the reserved molecule-runner-publish-* sub-pool (config.publish.yaml,
|
||||
# OUTSIDE the managed 1..20 range) so a merged fix's image build
|
||||
# starts immediately while PR-CI keeps the general pool.
|
||||
runs-on: publish
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing where `docker build`
|
||||
# fails deep in the process with a cryptic ECR auth error.
|
||||
- name: Verify Docker daemon access
|
||||
- name: Diagnose Docker daemon access
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
echo "::group::Docker daemon diagnosis"
|
||||
echo "Runner: ${HOSTNAME:-unknown}"
|
||||
docker_info="$(docker info 2>&1)" || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Runner: ${HOSTNAME:-unknown}"
|
||||
printf '%s\n' "${docker_info}"
|
||||
echo "::error::Check: (1) daemon is running, (2) runner user is in docker group, (3) sock permissions are 660+"
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "${docker_info}" | sed -n '1,5p'
|
||||
echo "Docker daemon OK"
|
||||
echo "--- Socket info ---"
|
||||
ls -la /var/run/docker.sock 2>/dev/null || echo "/var/run/docker.sock: not found"
|
||||
stat /var/run/docker.sock 2>/dev/null || true
|
||||
echo "--- User info ---"
|
||||
id
|
||||
echo "--- docker version ---"
|
||||
docker version 2>&1 || true
|
||||
echo "--- docker info (full) ---"
|
||||
docker info 2>&1 || echo "docker info failed: exit $?"
|
||||
echo "::endgroup::"
|
||||
|
||||
# Pre-clone manifest deps before docker build.
|
||||
@@ -140,18 +114,6 @@ jobs:
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Keep Buildx state inside the job temp dir. The publish runner's
|
||||
# inherited DOCKER_CONFIG can point at a host-owned ECR config path
|
||||
# (/home/hongming/.docker-ecr), which caused setup-buildx-action to
|
||||
# fail before image build with EACCES creating buildx/certs.
|
||||
- name: Prepare writable Docker config
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export DOCKER_CONFIG="$RUNNER_TEMP/docker-config"
|
||||
mkdir -p "$DOCKER_CONFIG/buildx/certs"
|
||||
echo "DOCKER_CONFIG=$DOCKER_CONFIG" >> "$GITHUB_ENV"
|
||||
docker buildx version
|
||||
|
||||
# Build + push platform image (inline ECR auth — mirrors the operator-host
|
||||
# approach; credentials come from GITHUB_SECRET_AWS_ACCESS_KEY_ID /
|
||||
# GITHUB_SECRET_AWS_SECRET_ACCESS_KEY in Gitea Actions).
|
||||
@@ -187,14 +149,9 @@ jobs:
|
||||
--push .
|
||||
|
||||
# Build + push tenant image (Go platform + Next.js canvas in one image).
|
||||
# Push the same build to the staging account too so fresh staging/E2E
|
||||
# tenants can pull without cross-account ECR reads. The staging ECR repo
|
||||
# policy trusts the primary SSOT-managed publisher principal; do not add
|
||||
# separate persistent staging AWS access keys here.
|
||||
- name: Build & push tenant image to ECR (staging-<sha> + staging-latest)
|
||||
env:
|
||||
TENANT_IMAGE_NAME: ${{ env.TENANT_IMAGE_NAME }}
|
||||
STAGING_TENANT_IMAGE_NAME: ${{ env.STAGING_TENANT_IMAGE_NAME }}
|
||||
TAG_SHA: staging-${{ steps.tags.outputs.sha }}
|
||||
TAG_LATEST: staging-latest
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
@@ -205,19 +162,8 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ECR_REGISTRY="${TENANT_IMAGE_NAME%%/*}"
|
||||
STAGING_ECR_REGISTRY="${STAGING_TENANT_IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${STAGING_ECR_REGISTRY}"
|
||||
|
||||
build_tags=(
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
--tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
--tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
)
|
||||
|
||||
docker buildx build \
|
||||
--file ./workspace-server/Dockerfile.tenant \
|
||||
--build-arg NEXT_PUBLIC_PLATFORM_URL= \
|
||||
@@ -226,186 +172,6 @@ jobs:
|
||||
--label "org.opencontainers.image.revision=${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--label "molecule.workflow.run_id=${GITHUB_RUN_ID}" \
|
||||
"${build_tags[@]}" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_SHA}" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}" \
|
||||
--push .
|
||||
|
||||
# bp-exempt: production deploy side-effect; merge is gated by CI / all-required and this job waits for push CI before acting.
|
||||
deploy-production:
|
||||
name: Production auto-deploy
|
||||
needs: build-and-push
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
# Side-effect deploy only; image publish success is the durable artifact. mc#1982
|
||||
continue-on-error: true
|
||||
# Publish/release lane (internal#462) — production deploy of a merged
|
||||
# fix; reserved capacity, never queued behind PR-CI.
|
||||
runs-on: publish
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
CP_URL: ${{ vars.PROD_CP_URL || 'https://api.moleculesai.app' }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
GITEA_TOKEN: ${{ secrets.PROD_AUTO_DEPLOY_CONTROL_TOKEN || secrets.AUTO_SYNC_TOKEN }}
|
||||
CI_STATUS_TIMEOUT_SECONDS: "3600"
|
||||
PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
|
||||
PROD_AUTO_DEPLOY_CANARY_SLUG: ${{ vars.PROD_AUTO_DEPLOY_CANARY_SLUG || 'hongming' }}
|
||||
PROD_AUTO_DEPLOY_SOAK_SECONDS: ${{ vars.PROD_AUTO_DEPLOY_SOAK_SECONDS || '60' }}
|
||||
PROD_AUTO_DEPLOY_BATCH_SIZE: ${{ vars.PROD_AUTO_DEPLOY_BATCH_SIZE || '3' }}
|
||||
PROD_AUTO_DEPLOY_DRY_RUN: ${{ vars.PROD_AUTO_DEPLOY_DRY_RUN || '' }}
|
||||
PROD_ALLOW_NON_PROD_CP_URL: ${{ vars.PROD_ALLOW_NON_PROD_CP_URL || '' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Build deploy plan
|
||||
id: plan
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/prod-auto-deploy.py plan > "$RUNNER_TEMP/prod-auto-deploy-plan.json"
|
||||
jq . "$RUNNER_TEMP/prod-auto-deploy-plan.json"
|
||||
enabled="$(jq -r '.enabled' "$RUNNER_TEMP/prod-auto-deploy-plan.json")"
|
||||
echo "enabled=$enabled" >> "$GITHUB_OUTPUT"
|
||||
if [ "$enabled" != "true" ]; then
|
||||
reason="$(jq -r '.disabled_reason' "$RUNNER_TEMP/prod-auto-deploy-plan.json")"
|
||||
echo "::notice::Production auto-deploy disabled: $reason"
|
||||
{
|
||||
echo "## Production auto-deploy skipped"
|
||||
echo ""
|
||||
echo "Reason: \`$reason\`"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret is required for production auto-deploy."
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is required so production deploy can wait for green CI."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Self-test production deploy helper
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -m pip install --quiet 'pytest==9.0.2' 'PyYAML==6.0.2'
|
||||
python3 -m pytest .gitea/scripts/tests/test_prod_auto_deploy.py -q
|
||||
python3 .gitea/scripts/lint-workflow-yaml.py --workflow-dir .gitea/workflows
|
||||
|
||||
- name: Wait for green main CI on this SHA
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/prod-auto-deploy.py wait-ci
|
||||
|
||||
- name: Call production CP redeploy-fleet
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/prod-auto-deploy.py assert-enabled
|
||||
PLAN="$RUNNER_TEMP/prod-auto-deploy-plan.json"
|
||||
TARGET_TAG="$(jq -r '.target_tag' "$PLAN")"
|
||||
|
||||
HTTP_RESPONSE="$RUNNER_TEMP/prod-redeploy-response.json"
|
||||
set +e
|
||||
python3 .gitea/scripts/prod-auto-deploy.py rollout \
|
||||
--plan "$PLAN" \
|
||||
--response "$HTTP_RESPONSE"
|
||||
ROLLOUT_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ ! -s "$HTTP_RESPONSE" ]; then
|
||||
jq -nc --arg error "rollout command exited $ROLLOUT_EXIT before writing a response" \
|
||||
'{ok:false, results:[], error:$error}' > "$HTTP_RESPONSE"
|
||||
fi
|
||||
jq '{ok, result_count: (.results // [] | length)}' "$HTTP_RESPONSE" || true
|
||||
|
||||
{
|
||||
echo "## Production auto-deploy"
|
||||
echo ""
|
||||
echo "**Commit:** \`${GITHUB_SHA:0:7}\`"
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo "| Slug | Phase | SSM Status | Exit | Healthz | On target | Error present |"
|
||||
echo "|------|-------|------------|------|---------|-----------|---------------|"
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.verified_on_target) | \((.error // "") != "") |"' "$HTTP_RESPONSE" || true
|
||||
# internal#724: stragglers are tenants enumerated but not proven
|
||||
# on the target build. Surface them loudly — a non-empty list
|
||||
# means the rollout did NOT fully land.
|
||||
STRAGGLERS="$(jq -r '(.stragglers // []) | join(", ")' "$HTTP_RESPONSE")"
|
||||
if [ -n "$STRAGGLERS" ]; then
|
||||
echo ""
|
||||
echo "### ⚠ Stragglers (NOT on target tag \`$TARGET_TAG\`)"
|
||||
echo ""
|
||||
echo "\`$STRAGGLERS\`"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
OK="$(jq -r '.ok' "$HTTP_RESPONSE")"
|
||||
if [ "$OK" != "true" ]; then
|
||||
STRAGGLERS="$(jq -r '(.stragglers // []) | join(", ")' "$HTTP_RESPONSE")"
|
||||
if [ -n "$STRAGGLERS" ]; then
|
||||
echo "::error::incomplete rollout — tenants not on target tag $TARGET_TAG: $STRAGGLERS"
|
||||
fi
|
||||
echo "::error::redeploy-fleet reported ok=false; production rollout halted."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$ROLLOUT_EXIT" -ne 0 ]; then
|
||||
echo "::error::redeploy-fleet rollout failed with exit code $ROLLOUT_EXIT."
|
||||
exit "$ROLLOUT_EXIT"
|
||||
fi
|
||||
|
||||
- name: Verify reachable tenants report this SHA
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
env:
|
||||
TENANT_DOMAIN: moleculesai.app
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RESP="$RUNNER_TEMP/prod-redeploy-response.json"
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::error::No tenants returned from redeploy-fleet; refusing to mark production deploy verified."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
UNHEALTHY_COUNT=0
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
healthz_ok="$(jq -r --arg slug "$slug" '.results[]? | select(.slug == $slug) | .healthz_ok' "$RESP" | tail -1)"
|
||||
if [ "$healthz_ok" != "true" ]; then
|
||||
echo "::error::$slug did not report healthz_ok=true in redeploy-fleet response."
|
||||
UNHEALTHY_COUNT=$((UNHEALTHY_COUNT + 1))
|
||||
continue
|
||||
fi
|
||||
url="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
body="$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$url" || true)"
|
||||
actual="$(echo "$body" | jq -r '.git_sha // ""' 2>/dev/null || echo "")"
|
||||
if [ -z "$actual" ]; then
|
||||
echo "::error::$slug did not return /buildinfo after deploy."
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
continue
|
||||
fi
|
||||
if [ "$actual" != "$GITHUB_SHA" ]; then
|
||||
echo "::error::$slug is stale: actual=${actual:0:7}, expected=${GITHUB_SHA:0:7}"
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
else
|
||||
echo "$slug: ${actual:0:7}"
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Buildinfo verification"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${GITHUB_SHA:0:7}\`"
|
||||
echo "Verified tenants: ${#SLUGS[@]}"
|
||||
echo "Stale tenants: $STALE_COUNT"
|
||||
echo "Unhealthy tenants: $UNHEALTHY_COUNT"
|
||||
echo "Unreachable tenants: $UNREACHABLE_COUNT"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$STALE_COUNT" -gt 0 ] || [ "$UNHEALTHY_COUNT" -gt 0 ] || [ "$UNREACHABLE_COUNT" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -9,22 +9,10 @@
|
||||
# Triggers on:
|
||||
# - `pull_request_target`: opened, synchronize, reopened
|
||||
# → initial status posts when PR opens / re-pushes
|
||||
# - `pull_request_review` types: [submitted]
|
||||
# → re-evaluate when a team member submits an APPROVE review so
|
||||
# the gate flips immediately (no wait for the next push or
|
||||
# slash-command). Verified live: sop-tier-check.yml uses this
|
||||
# same event and provably fires (produces
|
||||
# `sop-tier-check / tier-check (pull_request_review)` contexts).
|
||||
# The job-level `if:` guard checks
|
||||
# `github.event.review.state == 'APPROVED' || 'approved'` so
|
||||
# only APPROVE reviews run the evaluator; COMMENT and
|
||||
# REQUEST_CHANGES are skipped at the job level.
|
||||
# Branch-protection requires the `(pull_request_target)`
|
||||
# context variant, so the review-event path EXPLICITLY POSTS
|
||||
# the required context via the API. Trust boundary preserved
|
||||
# (BASE ref, no PR-head).
|
||||
# - comment refires are handled by `sop-checklist.yml` review-refire job
|
||||
# → `/qa-recheck` slash-command re-evaluates this gate.
|
||||
# - `issue_comment`: /qa-recheck slash-command on the PR
|
||||
# → manual re-fire after a QA reviewer clicks APPROVE
|
||||
# (Gitea 1.22.6 doesn't re-fire on pull_request_review, per
|
||||
# go-gitea/gitea#33700 + feedback_pull_request_review_no_refire)
|
||||
# Workflow name = `qa-review` ; job name = `approved`.
|
||||
# The job's own pass/fail conclusion publishes the status context
|
||||
# `qa-review / approved (<event>)` — NO `POST /statuses` call → NO
|
||||
@@ -97,26 +85,27 @@ name: qa-review
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
|
||||
jobs:
|
||||
# bp-exempt: PR review bot signal; required merge state is enforced by CI / all-required.
|
||||
approved:
|
||||
# Gate the job:
|
||||
# - On pull_request_target events: always run.
|
||||
# - On pull_request_review_approved events: run so the gate flips
|
||||
# immediately when a team member submits an APPROVE review.
|
||||
# Comment-triggered refires live in sop-checklist.yml review-refire job.
|
||||
# - On issue_comment events: only when it's a PR comment and the body
|
||||
# contains the slash-command. NO privilege gate at the step level
|
||||
# (RFC#324 v1.3 §A1.1): a non-collaborator's /qa-recheck is fine
|
||||
# because the eval is read-only and idempotent — re-running it
|
||||
# just re-confirms whether a real team-member APPROVE exists.
|
||||
if: |
|
||||
github.event_name == 'pull_request_target' ||
|
||||
(github.event_name == 'pull_request_review' &&
|
||||
(github.event.review.state == 'APPROVED' || github.event.review.state == 'approved'))
|
||||
(github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null &&
|
||||
startsWith(github.event.comment.body, '/qa-recheck'))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Privilege check (A1.1 — INFORMATIONAL log only, NOT a gate)
|
||||
@@ -130,7 +119,7 @@ jobs:
|
||||
# no comment.user.login so the step is a no-op skip there.
|
||||
if: github.event_name == 'issue_comment'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_TOKEN: ${{ secrets.RFC_324_TEAM_READ_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
login="${{ github.event.comment.user.login }}"
|
||||
@@ -160,81 +149,16 @@ jobs:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Evaluate qa-review
|
||||
id: eval
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_TOKEN: ${{ secrets.RFC_324_TEAM_READ_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
# PR number lives in different places per event:
|
||||
# pull_request_target → github.event.pull_request.number
|
||||
# issue_comment → github.event.issue.number
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: qa
|
||||
TEAM_ID: '20'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: bash .gitea/scripts/review-check.sh
|
||||
|
||||
- name: Post required status context on pull_request_review
|
||||
# Gitea Actions auto-publishes (pull_request_review) context
|
||||
# for this event, but branch-protection requires (pull_request_target).
|
||||
# We explicitly POST the BP-required context so the gate flips.
|
||||
# Trust boundary: same BASE-ref script result, no PR-head code.
|
||||
#
|
||||
# TOKEN FIX (RC 8326): uses STATUS_POST_TOKEN (CTO-granted,
|
||||
# msg d52cc72a). Dedicated narrow-scoped write:repository token
|
||||
# for the explicit status POST. Evaluator step stays on
|
||||
# SOP_TIER_CHECK_TOKEN (read-only) per deliberate security
|
||||
# separation: eval computes, POST writes, never the same cred.
|
||||
if: github.event_name == 'pull_request_review' && always()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.STATUS_POST_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
EVAL_OUTCOME: ${{ steps.eval.outcome }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
|
||||
prfile=$(mktemp)
|
||||
code=$(curl -sS -o "$prfile" -w '%{http_code}' -K "$authfile" \
|
||||
"https://${GITEA_HOST}/api/v1/repos/${REPO}/pulls/${PR_NUMBER}")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::GET /pulls/${PR_NUMBER} returned HTTP ${code}"
|
||||
rm -f "$prfile" "$authfile"
|
||||
exit 1
|
||||
fi
|
||||
head_sha=$(jq -r '.head.sha // ""' "$prfile")
|
||||
rm -f "$prfile"
|
||||
|
||||
if [ "$EVAL_OUTCOME" = "success" ]; then
|
||||
status_state="success"
|
||||
description="Approved via pull_request_review trigger"
|
||||
else
|
||||
status_state="failure"
|
||||
description="Review check failed via pull_request_review trigger"
|
||||
fi
|
||||
|
||||
body=$(jq -nc \
|
||||
--arg state "$status_state" \
|
||||
--arg context "qa-review / approved (pull_request_target)" \
|
||||
--arg description "$description" \
|
||||
'{state:$state, context:$context, description:$description}')
|
||||
|
||||
post_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
|
||||
-K "$authfile" -H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"https://${GITEA_HOST}/api/v1/repos/${REPO}/statuses/${head_sha}")
|
||||
|
||||
rm -f "$authfile"
|
||||
|
||||
if [ "$post_code" != "200" ] && [ "$post_code" != "201" ]; then
|
||||
echo "::error::POST /statuses/${head_sha} returned HTTP ${post_code}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::posted ${status_state} for context=\"qa-review / approved (pull_request_target)\" on sha=${head_sha}"
|
||||
|
||||
@@ -51,7 +51,6 @@ jobs:
|
||||
name: Audit Railway env vars for drift-prone pins
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 10
|
||||
|
||||
|
||||
@@ -9,17 +9,19 @@ name: redeploy-tenants-on-main
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - Dropped unsupported `workflow_run` (task #81).
|
||||
# - Later changed to manual-only after publish-workspace-server-image.yml
|
||||
# gained an integrated ordered production deploy job.
|
||||
# - ~~**Gitea workflow_run trigger limitation**~~ FIXED: replaced with
|
||||
# push+paths filter per this PR. Gitea 1.22.6 does not support
|
||||
# `workflow_run` (task #81). The push trigger fires on every
|
||||
# commit to publish-workspace-server-image.yml which is the
|
||||
# same signal (only successful runs commit to main).
|
||||
#
|
||||
|
||||
# Manual production tenant redeploy/rollback helper.
|
||||
# Auto-refresh prod tenant EC2s after every main merge.
|
||||
#
|
||||
# Why this workflow is manual-only: publish-workspace-server-image now owns
|
||||
# the ordered build -> push -> production auto-deploy sequence in one workflow.
|
||||
# A separate push-triggered redeploy workflow races before the new ECR image
|
||||
# exists and can paint main red with a false deployment failure.
|
||||
# Why this workflow exists: publish-workspace-server-image builds and
|
||||
# pushes a new platform-tenant :<sha> to ECR on every merge to main,
|
||||
# but running tenants pulled their image once at boot and never re-pull.
|
||||
# Users see stale code indefinitely.
|
||||
#
|
||||
# This workflow closes the gap by calling the control-plane admin
|
||||
# endpoint that performs a canary-first, batched, health-gated rolling
|
||||
@@ -32,61 +34,61 @@ name: redeploy-tenants-on-main
|
||||
# Gitea suspension migration. The staging-verify.yml promote step now
|
||||
# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
|
||||
#
|
||||
# Runtime ordering for automatic deploys now lives in
|
||||
# publish-workspace-server-image.yml:
|
||||
# 1. build-and-push creates new :staging-<sha> images in ECR.
|
||||
# 2. deploy-production waits for required push contexts on that SHA.
|
||||
# 3. deploy-production calls redeploy-fleet canary-first.
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
|
||||
# 2. This workflow fires via workflow_run, calls redeploy-fleet with
|
||||
# target_tag=staging-<sha>. No CDN propagation wait needed —
|
||||
# ECR image manifest is consistent immediately after push.
|
||||
# 3. Calls redeploy-fleet with canary_slug (if set) and a soak
|
||||
# period. Canary proves the image boots; batches follow.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: set PROD_MANUAL_REDEPLOY_TARGET_TAG as a repo/org
|
||||
# variable or secret, run workflow_dispatch, then unset it after the
|
||||
# rollback. That calls redeploy-fleet with target_tag=<value>,
|
||||
# re-pulling the pinned image on every tenant.
|
||||
# Rollback path: re-run this workflow with a specific SHA pinned via
|
||||
# the workflow_dispatch input. That calls redeploy-fleet with
|
||||
# target_tag=<sha>, re-pulling the older image on every tenant.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- '.gitea/workflows/publish-workspace-server-image.yml'
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize manual redeploys so two operator-triggered rollbacks do not
|
||||
# overlap and cause confusing per-tenant SSM state.
|
||||
# Serialize redeploys so two rapid main pushes' redeploys don't overlap
|
||||
# and cause confusing per-tenant SSM state. Without this, GitHub's
|
||||
# implicit workflow_run queueing would *probably* serialize them, but
|
||||
# the explicit block makes the invariant defensible. Mirrors the
|
||||
# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
|
||||
#
|
||||
# NOTE: cancel-in-progress: false removed (Rule 7 fix). Gitea 1.22.6
|
||||
# cancels queued runs regardless of this setting, so it provides no
|
||||
# actual protection. Each redeploy-fleet call is idempotent (canary-first
|
||||
# + batched + health-gated) so a cancelled predecessor is recovered
|
||||
# automatically by the next run.
|
||||
# cancel-in-progress: false → aborting a half-rolled-out fleet would
|
||||
# leave tenants stuck on whatever image they happened to be on when
|
||||
# cancelled. Better to finish the in-flight rollout before starting
|
||||
# the next one.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-main
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: production redeploy is a side-effect workflow, not a merge gate.
|
||||
redeploy:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399).
|
||||
# Production tenant redeploy — a deploy action, reserved capacity so
|
||||
# it never queues behind PR-CI. `publish` -> molecule-runner-publish-*.
|
||||
runs-on: publish
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
# NOTE (Gitea port): workflow_dispatch trigger dropped; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
# Rule 9 fix: keep the same operational kill switch surface as the
|
||||
# integrated auto-deploy workflow.
|
||||
PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
|
||||
steps:
|
||||
- name: Kill-switch guard
|
||||
# Rule 9 fix: exit fast if kill switch is set. No redeploy happens.
|
||||
if: env.PROD_AUTO_DEPLOY_DISABLED == 'true'
|
||||
run: |
|
||||
echo "::notice::Production auto-deploy disabled (PROD_AUTO_DEPLOY_DISABLED=true). Skipping redeploy."
|
||||
echo "To re-enable: unset the repo variable or set it to false."
|
||||
- name: Note on ECR propagation
|
||||
# ECR image manifests are consistent immediately after push — no
|
||||
# CDN cache to wait for. The old GHCR-based workflow had a 30s
|
||||
@@ -100,16 +102,21 @@ jobs:
|
||||
# tag) → used verbatim. Lets ops pin `latest` for emergency
|
||||
# rollback to last canary-verified digest, or pin a specific
|
||||
# `staging-<sha>` to roll back to a known-good build.
|
||||
# 2. Default → `staging-<short_head_sha>` for manual reruns from
|
||||
# the current default-branch SHA.
|
||||
# 2. Default → `staging-<short_head_sha>`. The just-published
|
||||
# digest. Bypasses the `:latest` retag path that's currently
|
||||
# dead (staging-verify soft-skips without canary fleet, so
|
||||
# the only thing retagging `:latest` today is the manual
|
||||
# promote-latest.yml — last run 2026-04-28). Auto-trigger
|
||||
# from workflow_run uses workflow_run.head_sha; manual
|
||||
# dispatch with no input falls through to github.sha.
|
||||
env:
|
||||
PROD_MANUAL_REDEPLOY_TARGET_TAG: ${{ vars.PROD_MANUAL_REDEPLOY_TARGET_TAG || secrets.PROD_MANUAL_REDEPLOY_TARGET_TAG || '' }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
INPUT_TAG: ${{ inputs.target_tag }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${PROD_MANUAL_REDEPLOY_TARGET_TAG:-}" ]; then
|
||||
echo "target_tag=$PROD_MANUAL_REDEPLOY_TARGET_TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Using operator-pinned tag from PROD_MANUAL_REDEPLOY_TARGET_TAG."
|
||||
if [ -n "${INPUT_TAG:-}" ]; then
|
||||
echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Using operator-pinned tag: $INPUT_TAG"
|
||||
else
|
||||
SHORT="${HEAD_SHA:0:7}"
|
||||
echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
|
||||
@@ -125,37 +132,19 @@ jobs:
|
||||
CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
CANARY_SLUG: ${{ vars.PROD_REDEPLOY_CANARY_SLUG || secrets.PROD_REDEPLOY_CANARY_SLUG || '' }}
|
||||
SOAK_SECONDS: ${{ vars.PROD_REDEPLOY_SOAK_SECONDS || secrets.PROD_REDEPLOY_SOAK_SECONDS || '' }}
|
||||
BATCH_SIZE: ${{ vars.PROD_REDEPLOY_BATCH_SIZE || secrets.PROD_REDEPLOY_BATCH_SIZE || '' }}
|
||||
DRY_RUN: ${{ vars.PROD_REDEPLOY_DRY_RUN || secrets.PROD_REDEPLOY_DRY_RUN || '' }}
|
||||
PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
case "${PROD_AUTO_DEPLOY_DISABLED,,}" in
|
||||
1|true|yes|on)
|
||||
echo "::notice::PROD_AUTO_DEPLOY_DISABLED is set; skipping production redeploy."
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
CANARY_SLUG="${CANARY_SLUG:-hongming}"
|
||||
SOAK_SECONDS="${SOAK_SECONDS:-60}"
|
||||
BATCH_SIZE="${BATCH_SIZE:-3}"
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. This caller redeploys
|
||||
# the entire prod fleet (canary + fan-out), no slug scoping, so
|
||||
# confirm:true is correct.
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
@@ -167,12 +156,11 @@ jobs:
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry,
|
||||
confirm: true
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag=$TARGET_TAG canary=$CANARY_SLUG soak_seconds=$SOAK_SECONDS batch_size=$BATCH_SIZE dry_run=$DRY_RUN"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
@@ -200,9 +188,7 @@ jobs:
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
# Rule 8 fix: redact raw CP response from CI logs. Print only
|
||||
# safe fields: ok boolean, result count, error presence (no content).
|
||||
jq '{ok, result_count: (.results | length), has_errors: (.results | any(.error != null))}' "$HTTP_RESPONSE" || echo "(jq parse failed)"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
# Pretty-print per-tenant results in the job summary so
|
||||
# ops can see which tenants were redeployed without drilling
|
||||
@@ -218,11 +204,9 @@ jobs:
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Errors |'
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
# Rule 8 fix: .error field redacted from CI logs/summary. Print only
|
||||
# presence boolean so ops know whether to look deeper.
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error != null) |"' "$HTTP_RESPONSE" || true
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
@@ -261,11 +245,13 @@ jobs:
|
||||
# fail the workflow, which is what `ok=true` should have
|
||||
# guaranteed all along.
|
||||
#
|
||||
# When the redeploy is triggered manually with a specific tag
|
||||
# (target_tag != "latest"), the expected SHA may not equal
|
||||
# ${{ github.sha }}.
|
||||
# When the redeploy was triggered by workflow_dispatch with a
|
||||
# specific tag (target_tag != "latest"), the expected SHA may
|
||||
# not equal ${{ github.sha }} — in that case we resolve via
|
||||
# GHCR's manifest. For workflow_run (default :latest) the
|
||||
# workflow_run.head_sha is the SHA that just published.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.sha }}
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
# Tenant subdomain template — slugs from the response are
|
||||
# appended. Production CP issues `<slug>.moleculesai.app`;
|
||||
@@ -279,10 +265,10 @@ jobs:
|
||||
if [ "$TARGET_TAG" != "latest" ] \
|
||||
&& [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
|
||||
&& [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
|
||||
# Manual redeploy with a pinned tag that isn't the head
|
||||
# workflow_dispatch with a pinned tag that isn't the head
|
||||
# SHA — operator is rolling back / pinning. Skip the
|
||||
# verification because we don't have the expected SHA in
|
||||
# this context (would need to inspect the ECR
|
||||
# this context (would need to crane-inspect the GHCR
|
||||
# manifest, which is a follow-up). Failing-open here is
|
||||
# safe: the operator chose the tag deliberately.
|
||||
#
|
||||
|
||||
@@ -73,14 +73,9 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: post-merge staging redeploy side effect; CI / all-required gates source changes.
|
||||
redeploy:
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399).
|
||||
# Post-merge staging redeploy — a deploy action, reserved capacity.
|
||||
# `publish` -> molecule-runner-publish-* sub-pool.
|
||||
runs-on: publish
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
@@ -123,11 +118,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. Staging IS the
|
||||
# canary, no slug scoping; this rolls the entire staging fleet,
|
||||
# so confirm:true is correct.
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
@@ -139,8 +129,7 @@ jobs:
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry,
|
||||
confirm: true
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
|
||||
@@ -41,7 +41,6 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: review tooling regression suite; CI / all-required is the required aggregate.
|
||||
test:
|
||||
name: review-check.sh regression tests
|
||||
runs-on: ubuntu-latest
|
||||
@@ -54,7 +53,6 @@ jobs:
|
||||
# runners with internet access to package mirrors). Falls back to GitHub
|
||||
# binary download. GitHub releases may be blocked on some runner networks
|
||||
# (infra#241 follow-up).
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
if apt-get update -qq && apt-get install -y -qq jq; then
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# DEPRECATED — superseded by `.gitea/workflows/sop-checklist.yml`.
|
||||
#
|
||||
# The review-refire logic (qa/security/tier slash-command dispatch) has been
|
||||
# merged into sop-checklist.yml as the `review-refire` job. This workflow
|
||||
# is kept as a no-op stub to avoid a gap during the transition window where
|
||||
# this file may be deleted while sop-checklist.yml has not yet been merged.
|
||||
#
|
||||
# After sop-checklist.yml lands, this file will be deleted (issue #1280).
|
||||
#
|
||||
# Historical behavior (superseded):
|
||||
# Gitea 1.22 queues one run per workflow subscribed to `issue_comment` before
|
||||
# evaluating job-level `if:`. Previously this workflow was the single
|
||||
# non-SOP comment subscriber for qa/security/tier refire slash commands.
|
||||
|
||||
name: review-refire-comments
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.repository }}-${{ github.workflow }}-${{ github.event.issue.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# No-op stub — all refire logic moved to sop-checklist.yml review-refire job.
|
||||
# Kept to avoid transition gap; will be deleted after sop-checklist.yml merges.
|
||||
dispatch:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deprecated — refire logic moved to sop-checklist.yml
|
||||
run: |
|
||||
echo "::warning::review-refire-comments.yml is deprecated. Refire logic is now in sop-checklist.yml review-refire job. This workflow is a no-op stub pending deletion (issue #1280)."
|
||||
exit 0
|
||||
@@ -0,0 +1,100 @@
|
||||
name: Runtime Pin Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-pin-compat.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and
|
||||
# `workflow_dispatch:` (no inputs, but the trigger itself is
|
||||
# parser-rejected when inputs are absent in some Gitea 1.22.x
|
||||
# builds; safest to drop entirely — manual runs go via cron-trigger
|
||||
# bump or push-with-paths-filter).
|
||||
# - on.paths references .gitea/workflows/runtime-pin-compat.yml (this
|
||||
# file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
|
||||
# recurring (controlplane#253). The original failure mode:
|
||||
# 1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
|
||||
# requires_dist metadata (incorrect — it actually imports
|
||||
# a2a.server.routes which only exists in a2a-sdk 1.0+)
|
||||
# 2. `pip install molecule-ai-workspace-runtime` resolved cleanly
|
||||
# 3. `from molecule_runtime.main import main_sync` raised ImportError
|
||||
# 4. Every tenant workspace crashed; the canary tenant caught it but
|
||||
# only after 5 hours of degraded staging
|
||||
#
|
||||
# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
|
||||
# top of `workspace/requirements.txt` and smoke-imports. Catches:
|
||||
# - Upstream PyPI yanks
|
||||
# - Bad re-releases of molecule-ai-workspace-runtime
|
||||
# - Already-shipped wheels that stop importing because a transitive
|
||||
# dep moved underneath
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
# Narrow filter: pypi-latest is sensitive only to changes that
|
||||
# affect what we're INSTALLING (requirements.txt) or WHAT THE
|
||||
# CHECK ITSELF DOES (this workflow file). Edits to workspace/
|
||||
# source code don't change what's on PyPI right now, so they
|
||||
# don't change this gate's verdict.
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
# Daily catch for upstream PyPI publishes that break the pin combo
|
||||
# without any change in our repo (e.g. someone re-yanks an a2a-sdk
|
||||
# release or molecule-ai-workspace-runtime publishes a bad bump).
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # 06:00 PT
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pypi-latest-install:
|
||||
name: PyPI-latest install + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install runtime + workspace requirements
|
||||
# Install order is load-bearing: install the runtime FIRST so pip
|
||||
# honors whatever a2a-sdk constraint the runtime metadata declares
|
||||
# (this is the surface that broke in 2026-04-24 — runtime declared
|
||||
# `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
|
||||
# of workspace/requirements.txt then upgrades a2a-sdk to the
|
||||
# constraint our runtime image actually pins. The import smoke
|
||||
# below verifies the upgraded combination is consistent.
|
||||
run: |
|
||||
python -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install --upgrade pip
|
||||
/tmp/venv/bin/pip install molecule-ai-workspace-runtime
|
||||
/tmp/venv/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import — fail if metadata declares deps that don't satisfy real imports
|
||||
# WORKSPACE_ID is validated at import time by platform_auth.py — EC2
|
||||
# user-data sets it from the cloud-init template; set a placeholder
|
||||
# here so the import smoke doesn't trip on the env-var guard.
|
||||
env:
|
||||
WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
|
||||
run: |
|
||||
/tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"
|
||||
@@ -0,0 +1,139 @@
|
||||
name: Runtime PR-Built Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-prbuild-compat.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and `workflow_dispatch:`
|
||||
# (Gitea 1.22.6 parser-rejects workflow_dispatch with inputs and is
|
||||
# finicky without them).
|
||||
# - `dorny/paths-filter@v4` replaced with inline `git diff` (per PR#372
|
||||
# pattern for ci.yml port).
|
||||
# - on.paths references .gitea/workflows/runtime-prbuild-compat.yml.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on every job (RFC §1 contract).
|
||||
#
|
||||
# Companion to `runtime-pin-compat.yml`. That workflow tests what's
|
||||
# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
|
||||
# PUBLISHED if THIS PR merges.
|
||||
#
|
||||
# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
|
||||
# wheel" job to the original runtime-pin-compat.yml, but both jobs
|
||||
# shared a `paths:` filter that was the union of their needs
|
||||
# (`workspace/**`). That meant the PyPI-latest job ran on every doc
|
||||
# edit even though the upstream PyPI artifact can't change with our
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter that matches the inputs it actually depends on.
|
||||
#
|
||||
# Catches the failure mode where a PR adds an import requiring a newer
|
||||
# SDK than `workspace/requirements.txt` pins:
|
||||
# 1. Pip resolves the existing PyPI wheel + the old SDK pin -> smoke
|
||||
# passes (it imports the OLD main.py from the wheel, not the PR's
|
||||
# new main.py).
|
||||
# 2. Merge -> publish-runtime.yml ships a wheel WITH the new import.
|
||||
# 3. Tenant images redeploy -> all crash on first boot with ImportError.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
# event_name + sha keeps PR sync and the subsequent staging push on the
|
||||
# same SHA from cancelling each other (per feedback_concurrency_group_per_sha).
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
wheel: ${{ steps.decide.outputs.wheel }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
run: |
|
||||
# Inline replacement for dorny/paths-filter — same pattern
|
||||
# PR#372's ci.yml port used. Diffs against the PR base or the
|
||||
# previous push SHA, then matches against the wheel-relevant
|
||||
# path set.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
# New branch or no previous SHA: treat as wheel-relevant.
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace/|scripts/build_runtime_package\.py$|scripts/wheel_smoke\.py$|\.gitea/workflows/runtime-prbuild-compat\.yml$)'; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "wheel=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `PR-built wheel + import smoke`. Real work is
|
||||
# gated per-step on `needs.detect-changes.outputs.wheel`.
|
||||
local-build-install:
|
||||
needs: detect-changes
|
||||
name: PR-built wheel + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.wheel != 'true'
|
||||
run: |
|
||||
echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
|
||||
echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install build tooling
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: pip install build
|
||||
- name: Build wheel from PR source (mirrors publish-runtime.yml)
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Use a fixed test version so the wheel filename is predictable.
|
||||
# Doesn't reach PyPI — this build is local-only for the smoke.
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "0.0.0.dev0+pin-compat" \
|
||||
--out /tmp/runtime-build
|
||||
cd /tmp/runtime-build && python -m build
|
||||
- name: Install built wheel + workspace requirements
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: |
|
||||
python -m venv /tmp/venv-built
|
||||
/tmp/venv-built/bin/pip install --upgrade pip
|
||||
/tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
|
||||
/tmp/venv-built/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import the PR-built wheel
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Same script publish-runtime.yml runs against the to-be-PyPI wheel.
|
||||
run: |
|
||||
/tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
@@ -57,7 +57,6 @@ jobs:
|
||||
name: Detect SECRET_PATTERNS drift
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
|
||||
@@ -30,11 +30,6 @@ jobs:
|
||||
scan:
|
||||
name: Scan diff for credential-shaped strings
|
||||
runs-on: ubuntu-latest
|
||||
# Hard CI gate — must complete or the PR is unmergable. 10-minute ceiling
|
||||
# is generous for a diff-scan against a single SHA. If this times out, the
|
||||
# runner is frozen and holding a slot — the step timeout triggers clean
|
||||
# failure, releasing the runner for the next job.
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
@@ -138,14 +133,6 @@ jobs:
|
||||
[ -z "$f" ] && continue
|
||||
[ "$f" = "$SELF_GITHUB" ] && continue
|
||||
[ "$f" = "$SELF_GITEA" ] && continue
|
||||
# Test-fixture exclude (internal#425): the secrets-detector's OWN
|
||||
# unit-test corpus deliberately embeds credential-SHAPED example
|
||||
# strings to exercise the detector. Verified 2026-05-18 synthetic
|
||||
# (fabricated ghp_* fixtures, not real). Without this the scanner
|
||||
# self-trips on its own fixtures and fail-closes every deploy.
|
||||
# Same rationale as the SELF_* excludes above; gate NOT weakened
|
||||
# (all other paths still fully scanned).
|
||||
[ "$f" = "workspace-server/internal/secrets/patterns_test.go" ] && continue
|
||||
if [ -n "$DIFF_RANGE" ]; then
|
||||
ADDED=$(git diff --no-color --unified=0 "$BASE" "$HEAD" -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
|
||||
else
|
||||
|
||||
@@ -6,44 +6,28 @@
|
||||
#
|
||||
# See `qa-review.yml` header for the full A1-α / A1.1 / A4 / A5 design
|
||||
# rationale; everything below is identical in shape.
|
||||
#
|
||||
# A1-α addendum (internal#760): review-event trigger added so the security
|
||||
# gate flips immediately when a team member submits an APPROVE review.
|
||||
# Uses `pull_request_review` types: [submitted] — verified live via
|
||||
# sop-tier-check.yml which provably fires this event (produces
|
||||
# `sop-tier-check / tier-check (pull_request_review)` contexts).
|
||||
# The job-level `if:` guard checks
|
||||
# `github.event.review.state == 'APPROVED' || 'approved'` so only APPROVE
|
||||
# reviews run the evaluator; COMMENT and REQUEST_CHANGES are skipped at
|
||||
# the job level. Branch-protection requires the `(pull_request_target)`
|
||||
# context variant, so the review-event path EXPLICITLY POSTS the required
|
||||
# context via the API. Trust boundary preserved (BASE ref, no PR-head).
|
||||
|
||||
name: security-review
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
|
||||
jobs:
|
||||
# bp-exempt: PR security review bot signal; required merge state is enforced by CI / all-required.
|
||||
approved:
|
||||
# Gate the job:
|
||||
# - On pull_request_target events: always run.
|
||||
# - On pull_request_review_approved events: run so the gate flips
|
||||
# immediately when a team member submits an APPROVE review.
|
||||
# Comment-triggered refires live in sop-checklist.yml review-refire job.
|
||||
# See qa-review.yml header for full A1-α / A1.1 (v1.3 — informational
|
||||
# log only, NOT a gate) / A4 / A5 design rationale.
|
||||
if: |
|
||||
github.event_name == 'pull_request_target' ||
|
||||
(github.event_name == 'pull_request_review' &&
|
||||
(github.event.review.state == 'APPROVED' || github.event.review.state == 'approved'))
|
||||
(github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null &&
|
||||
startsWith(github.event.comment.body, '/security-recheck'))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Privilege check (A1.1 — INFORMATIONAL log only, NOT a gate)
|
||||
@@ -52,7 +36,7 @@ jobs:
|
||||
# so re-running on a non-collaborator comment is harmless.
|
||||
if: github.event_name == 'issue_comment'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_TOKEN: ${{ secrets.RFC_324_TEAM_READ_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
login="${{ github.event.comment.user.login }}"
|
||||
@@ -76,78 +60,13 @@ jobs:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Evaluate security-review
|
||||
id: eval
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_TOKEN: ${{ secrets.RFC_324_TEAM_READ_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: security
|
||||
TEAM_ID: '21'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: bash .gitea/scripts/review-check.sh
|
||||
|
||||
- name: Post required status context on pull_request_review
|
||||
# Gitea Actions auto-publishes (pull_request_review) context
|
||||
# for this event, but branch-protection requires (pull_request_target).
|
||||
# We explicitly POST the BP-required context so the gate flips.
|
||||
# Trust boundary: same BASE-ref script result, no PR-head code.
|
||||
#
|
||||
# TOKEN FIX (RC 8326): uses STATUS_POST_TOKEN (CTO-granted,
|
||||
# msg d52cc72a). Dedicated narrow-scoped write:repository token
|
||||
# for the explicit status POST. Evaluator step stays on
|
||||
# SOP_TIER_CHECK_TOKEN (read-only) per deliberate security
|
||||
# separation: eval computes, POST writes, never the same cred.
|
||||
if: github.event_name == 'pull_request_review' && always()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.STATUS_POST_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
EVAL_OUTCOME: ${{ steps.eval.outcome }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
|
||||
prfile=$(mktemp)
|
||||
code=$(curl -sS -o "$prfile" -w '%{http_code}' -K "$authfile" \
|
||||
"https://${GITEA_HOST}/api/v1/repos/${REPO}/pulls/${PR_NUMBER}")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::GET /pulls/${PR_NUMBER} returned HTTP ${code}"
|
||||
rm -f "$prfile" "$authfile"
|
||||
exit 1
|
||||
fi
|
||||
head_sha=$(jq -r '.head.sha // ""' "$prfile")
|
||||
rm -f "$prfile"
|
||||
|
||||
if [ "$EVAL_OUTCOME" = "success" ]; then
|
||||
status_state="success"
|
||||
description="Approved via pull_request_review trigger"
|
||||
else
|
||||
status_state="failure"
|
||||
description="Review check failed via pull_request_review trigger"
|
||||
fi
|
||||
|
||||
body=$(jq -nc \
|
||||
--arg state "$status_state" \
|
||||
--arg context "security-review / approved (pull_request_target)" \
|
||||
--arg description "$description" \
|
||||
'{state:$state, context:$context, description:$description}')
|
||||
|
||||
post_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
|
||||
-K "$authfile" -H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"https://${GITEA_HOST}/api/v1/repos/${REPO}/statuses/${head_sha}")
|
||||
|
||||
rm -f "$authfile"
|
||||
|
||||
if [ "$post_code" != "200" ] && [ "$post_code" != "201" ]; then
|
||||
echo "::error::POST /statuses/${head_sha} returned HTTP ${post_code}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::posted ${status_state} for context=\"security-review / approved (pull_request_target)\" on sha=${head_sha}"
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
# sop-checklist-gate — peer-ack merge gate for SOP-checklist items.
|
||||
#
|
||||
# RFC#351 Step 2 of 6 (implementation MVP).
|
||||
#
|
||||
# === DESIGN ===
|
||||
#
|
||||
# Goal: each PR must answer 7 SOP-checklist questions in its body,
|
||||
# and each item must have at least one /sop-ack <slug> comment from
|
||||
# a non-author peer in the required team. BP requires the
|
||||
# `sop-checklist / all-items-acked (pull_request)` status to merge.
|
||||
#
|
||||
# Triggers:
|
||||
# - `pull_request_target`: opened, edited, synchronize, reopened
|
||||
# → fires when PR opens, body is edited (refire — RFC#351 §4),
|
||||
# or new code is pushed (head.sha changes → stale status would
|
||||
# be auto-discarded by BP via dismiss_stale_reviews, but the
|
||||
# status itself is per-SHA so we re-post on the new head).
|
||||
# - `issue_comment`: created, edited, deleted
|
||||
# → fires on any new comment so /sop-ack / /sop-revoke take
|
||||
# effect immediately (Gitea 1.22.6 doesn't refire on
|
||||
# pull_request_review per feedback_pull_request_review_no_refire,
|
||||
# so issue_comment is the canonical refire channel).
|
||||
#
|
||||
# Trust boundary (mirrors RFC#324 §A4 + sop-tier-check security note):
|
||||
# `pull_request_target` (not `pull_request`) — workflow def is loaded
|
||||
# from BASE branch, so a PR cannot rewrite this workflow to exfiltrate
|
||||
# the token. The `actions/checkout` step pins `ref: base.sha` so the
|
||||
# script ALSO comes from BASE. PR-HEAD code is never executed in the
|
||||
# runner.
|
||||
#
|
||||
# Token scope:
|
||||
# - read:repository, read:organization for PR + comments + team probes
|
||||
# - write:repository for POST /statuses/{sha}
|
||||
# - The token owner MUST be a member of every team referenced by the
|
||||
# config's required_teams (else /teams/{id}/members/{login} returns
|
||||
# 403 — see review-check.sh same-gotcha doc). For the MVP we use
|
||||
# the dev-lead token (a member of engineers, managers, qa, security)
|
||||
# via a repo secret `SOP_CHECKLIST_GATE_TOKEN`. Provisioning of that
|
||||
# secret is a follow-up authorization step (separate from this PR).
|
||||
#
|
||||
# Failure mode: tier-aware (RFC#351 open question 2):
|
||||
# - tier:high → state=failure (hard-fail; BP blocks merge)
|
||||
# - tier:medium → state=failure (hard-fail; same)
|
||||
# - tier:low → state=pending (soft-fail; BP can choose to require
|
||||
# this context or skip for low-tier PRs)
|
||||
# - missing/no-tier → state=failure (default-mode: hard — never lower
|
||||
# the bar per feedback_fix_root_not_symptom)
|
||||
#
|
||||
# Slash-command contract (RFC#351 v1 + §A1.1-style notes from RFC#324):
|
||||
#
|
||||
# /sop-ack <slug-or-numeric-alias> [optional note]
|
||||
# — register a peer-ack for one checklist item.
|
||||
# — slug accepts kebab-case, snake_case, or natural-spaces
|
||||
# (all normalize to canonical kebab-case).
|
||||
# — numeric 1..7 maps via config.items[*].numeric_alias.
|
||||
# — most-recent (user, slug) directive wins.
|
||||
#
|
||||
# /sop-revoke <slug-or-numeric-alias> [reason]
|
||||
# — invalidate the commenter's own prior /sop-ack for this slug.
|
||||
# — does NOT affect other peers' acks on the same slug.
|
||||
# — most-recent (user, slug) directive wins, so a later /sop-ack
|
||||
# re-restores the ack.
|
||||
#
|
||||
# The eval is read-only + idempotent (read PR + comments + team
|
||||
# membership, compute, post status). Re-running on any event is safe —
|
||||
# the new status overwrites the previous one for the same context.
|
||||
|
||||
name: sop-checklist-gate
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, synchronize, reopened]
|
||||
issue_comment:
|
||||
types: [created, edited, deleted]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
# NOTE: `statuses: write` is the GitHub-Actions name for POST /statuses.
|
||||
# Gitea 1.22.6 may not gate on this permission key (it just checks the
|
||||
# token), but listing it explicitly documents intent for the next
|
||||
# platform-version upgrade.
|
||||
statuses: write
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
# Run on pull_request_target events always. On issue_comment events,
|
||||
# only when the comment is on a PR (issue_comment fires for issues
|
||||
# too) and the body contains one of the slash-commands.
|
||||
if: |
|
||||
github.event_name == 'pull_request_target' ||
|
||||
(github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null &&
|
||||
(contains(github.event.comment.body, '/sop-ack') ||
|
||||
contains(github.event.comment.body, '/sop-revoke')))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out BASE ref (trust boundary — never PR-head)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# For pull_request_target, the default branch is the trust
|
||||
# anchor. For issue_comment the PR base may differ from the
|
||||
# default branch (PR targeting `staging`), so we use the
|
||||
# default-branch ref explicitly — same approach as
|
||||
# qa-review.yml so the script source is always trusted.
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Run sop-checklist-gate
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_CHECKLIST_GATE_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO_NAME: ${{ github.event.repository.name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/sop-checklist-gate.py \
|
||||
--owner "$OWNER" \
|
||||
--repo "$REPO_NAME" \
|
||||
--pr "$PR_NUMBER" \
|
||||
--config .gitea/sop-checklist-config.yaml \
|
||||
--gitea-host git.moleculesai.app
|
||||
@@ -1,225 +0,0 @@
|
||||
# sop-checklist — peer-ack merge gate for SOP-checklist items.
|
||||
#
|
||||
# RFC#351 Step 2 of 6 (implementation MVP).
|
||||
#
|
||||
# === CONSOLIDATION (issue #1280) ===
|
||||
#
|
||||
# This workflow is the SINGLE `issue_comment` subscriber — the logic from
|
||||
# `review-refire-comments.yml` has been merged in. Before this change:
|
||||
# - sop-checklist.yml (pre-2026-05-16) → issue_comment:[created,edited,deleted] → runner slot used, job no-oped
|
||||
# - review-refire-comments.yml → issue_comment:[created] → runner slot used, job no-oped
|
||||
# → every non-refire comment occupied 2 runner slots for ~800 s each
|
||||
# (~650 no-op runs/day, ~1,300 runner-slot-occupancy-hours/day).
|
||||
#
|
||||
# Fix (PR #1345 / issue #1280):
|
||||
# - ONE workflow, ONE issue_comment:[created] subscription (no edited/deleted)
|
||||
# - all-items-acked job: pull_request_target OR sop slash-command comments
|
||||
# - review-refire job: qa/security/tier refire slash commands
|
||||
# → ~50% reduction in comment-triggered runner occupancy vs pre-fix.
|
||||
#
|
||||
# Trust boundary (mirrors RFC#324 §A4 + sop-tier-check security note):
|
||||
# `pull_request_target` (not `pull_request`) — workflow def is loaded
|
||||
# from BASE branch, so a PR cannot rewrite this workflow to exfiltrate
|
||||
# the token. The `actions/checkout` step pins `ref: base.sha` so the
|
||||
# script ALSO comes from BASE. PR-HEAD code is never executed in the
|
||||
# runner.
|
||||
#
|
||||
# Token scope:
|
||||
# - read:repository, read:organization for PR + comments + team probes
|
||||
# - write:repository for POST /statuses/{sha}
|
||||
# - The token owner MUST be a member of every team referenced by the
|
||||
# config's required_teams (else /teams/{id}/members/{login} returns
|
||||
# 403 — see review-check.sh same-gotcha doc). For the MVP we use
|
||||
# the dev-lead token (a member of engineers, managers, qa, security)
|
||||
# via a repo secret `SOP_CHECKLIST_GATE_TOKEN`. Provisioning of that
|
||||
# secret is a follow-up authorization step (separate from this PR).
|
||||
#
|
||||
# Failure mode: tier-aware (RFC#351 open question 2):
|
||||
# - tier:high → state=failure (hard-fail; BP blocks merge)
|
||||
# - tier:medium → state=failure (hard-fail; same)
|
||||
# - tier:low → state=pending (soft-fail; BP can choose to require
|
||||
# this context or skip for low-tier PRs)
|
||||
# - missing/no-tier → state=failure (default-mode: hard — never lower
|
||||
# the bar per feedback_fix_root_not_symptom)
|
||||
#
|
||||
# Slash-command contract (RFC#351 v1 + §A1.1-style notes from RFC#324):
|
||||
#
|
||||
# /sop-ack <slug-or-numeric-alias> [optional note]
|
||||
# — register a peer-ack for one checklist item.
|
||||
# — slug accepts kebab-case, snake_case, or natural-spaces
|
||||
# (all normalized to canonical kebab-case).
|
||||
# — numeric 1..7 maps via config.items[*].numeric_alias.
|
||||
# — most-recent (user, slug) directive wins.
|
||||
#
|
||||
# /sop-revoke <slug-or-numeric-alias> [reason]
|
||||
# — invalidate the commenter's own prior /sop-ack for this slug.
|
||||
# — does NOT affect other peers' acks on the same slug.
|
||||
# — most-recent (user, slug) directive wins, so a later /sop-ack
|
||||
# re-restores the ack.
|
||||
#
|
||||
# /sop-n/a <gate> [reason]
|
||||
# — declare a gate (qa-review, security-review) N/A.
|
||||
# — see sop-checklist-config.yaml n/a_gates section.
|
||||
#
|
||||
# /qa-recheck /security-recheck /refire-tier-check
|
||||
# — refire the corresponding status check on the PR head.
|
||||
#
|
||||
# The eval is read-only + idempotent (read PR + comments + team
|
||||
# membership, compute, post status). Re-running on any event is safe —
|
||||
# the new status overwrites the previous one for the same context.
|
||||
|
||||
name: sop-checklist
|
||||
|
||||
# Cancel any in-progress runs for the same PR to prevent
|
||||
# stale runs from overwriting newer status contexts.
|
||||
concurrency:
|
||||
group: ${{ github.repository }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# bp-required: yes ← emits sop-checklist / all-items-acked (pull_request)
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, synchronize, reopened, labeled, unlabeled]
|
||||
issue_comment:
|
||||
types: [created] # NOT [created, edited, deleted] — Gitea 1.22.6 holds a runner slot
|
||||
# at job-parsing time, before job-level if: guards run. edited/deleted events
|
||||
# occupied ~1,300 runner-slot-hours/day on this workflow alone during the
|
||||
# 2026-05-16 freeze. Per PR #1345 fix.
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
secrets: read
|
||||
|
||||
jobs:
|
||||
# sop-checklist gate: runs on PR lifecycle events OR sop slash commands.
|
||||
# All other comment types (no-op text comments) no longer assign a runner
|
||||
# because this job's if: guard short-circuits before runner assignment.
|
||||
all-items-acked:
|
||||
if: |
|
||||
github.event_name == 'pull_request_target' ||
|
||||
(github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null &&
|
||||
(contains(github.event.comment.body, '/sop-ack') ||
|
||||
contains(github.event.comment.body, '/sop-revoke') ||
|
||||
contains(github.event.comment.body, '/sop-n/a')))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out BASE ref (trust boundary — never PR-head)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# For pull_request_target, the default branch is the trust
|
||||
# anchor. For issue_comment the PR base may differ from the
|
||||
# default branch (PR targeting `staging`), so we use the
|
||||
# default-branch ref explicitly — same approach as
|
||||
# qa-review.yml so the script source is always trusted.
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Run sop-checklist
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_CHECKLIST_GATE_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO_NAME: ${{ github.event.repository.name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/sop-checklist.py \
|
||||
--owner "$OWNER" \
|
||||
--repo "$REPO_NAME" \
|
||||
--pr "$PR_NUMBER" \
|
||||
--config .gitea/sop-checklist-config.yaml \
|
||||
--gitea-host git.moleculesai.app
|
||||
|
||||
# bp-exempt: informational refire handler, not a merge gate. Emits
|
||||
# qa-review/security-review status updates on /qa-recheck et al slash commands.
|
||||
review-refire:
|
||||
if: |
|
||||
github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Classify comment
|
||||
id: classify
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo "run_qa=false"
|
||||
echo "run_security=false"
|
||||
echo "run_tier=false"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
first_line=$(printf '%s\n' "$COMMENT_BODY" | sed -n '1p')
|
||||
case "$first_line" in
|
||||
/qa-recheck*)
|
||||
echo "run_qa=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
/security-recheck*)
|
||||
echo "run_security=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
/refire-tier-check*)
|
||||
echo "run_tier=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
echo "::notice::no supported review refire slash command; no-op"
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Check out BASE ref for trusted scripts
|
||||
if: |
|
||||
steps.classify.outputs.run_qa == 'true' ||
|
||||
steps.classify.outputs.run_security == 'true' ||
|
||||
steps.classify.outputs.run_tier == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Refire qa-review status
|
||||
if: steps.classify.outputs.run_qa == 'true'
|
||||
env:
|
||||
# Evaluator (review-check.sh + GET /pulls) stays on read-scoped token.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
# Explicit POST /statuses uses narrow-scoped write:repository token.
|
||||
STATUS_POST_TOKEN: ${{ secrets.STATUS_POST_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: qa
|
||||
TEAM_ID: '20'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
.gitea/scripts/review-refire-status.sh
|
||||
|
||||
- name: Refire security-review status
|
||||
if: steps.classify.outputs.run_security == 'true'
|
||||
env:
|
||||
# Evaluator (review-check.sh + GET /pulls) stays on read-scoped token.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
# Explicit POST /statuses uses narrow-scoped write:repository token.
|
||||
STATUS_POST_TOKEN: ${{ secrets.STATUS_POST_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: security
|
||||
TEAM_ID: '21'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
.gitea/scripts/review-refire-status.sh
|
||||
|
||||
- name: Refire sop-tier-check status
|
||||
if: steps.classify.outputs.run_tier == 'true'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
SOP_DEBUG: '0'
|
||||
run: bash .gitea/scripts/sop-tier-refire.sh
|
||||
@@ -28,16 +28,15 @@
|
||||
#
|
||||
# Environment variables:
|
||||
# SOP_DEBUG=1 — per-API-call diagnostic lines. Default: off.
|
||||
# SOP_LEGACY_CHECK=1 — revert to OR-gate for this run. Intended for
|
||||
# emergency use only; burn-in window closed
|
||||
# 2026-05-17 (internal#189 Phase 1).
|
||||
# SOP_LEGACY_CHECK=1 — revert to OR-gate for this run. Grace window
|
||||
# for PRs in-flight when AND-composition deployed.
|
||||
# Burn-in: remove after 2026-05-17 (7-day window).
|
||||
#
|
||||
# BURN-IN CLOSED 2026-05-17 (internal#189 Phase 1): The 7-day burn-in
|
||||
# window closed. continue-on-error: true has been removed from the
|
||||
# tier-check job; AND-composition is now fully enforced. If you need
|
||||
# to temporarily re-introduce a mask, file a tracker and follow the
|
||||
# mc#1982 protocol (Tier 2e lint requires a current tracker within
|
||||
# 2 lines of any continue-on-error: true).
|
||||
# BURN-IN NOTE (internal#189 Phase 1): continue-on-error: true is set on
|
||||
# the tier-check job below. This prevents AND-composition from blocking
|
||||
# PRs during the 7-day burn-in. After 2026-05-17:
|
||||
# 1. Remove `continue-on-error: true` from this job block.
|
||||
# 2. Update this BURN-IN NOTE comment to mark the window closed.
|
||||
|
||||
name: sop-tier-check
|
||||
|
||||
@@ -61,17 +60,15 @@ on:
|
||||
pull_request_review:
|
||||
types: [submitted, dismissed, edited]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.repository }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
tier-check:
|
||||
runs-on: ubuntu-latest
|
||||
# BURN-IN: continue-on-error prevents AND-composition from blocking
|
||||
# PRs during the 7-day window. Remove after 2026-05-17 (internal#189).
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
secrets: read
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -92,7 +89,6 @@ jobs:
|
||||
# runners). The sop-tier-check script has its own fallback as a
|
||||
# third line of defense. continue-on-error: true ensures this step
|
||||
# failing does not block the job.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# apt-get is the primary method — Ubuntu package mirrors are reliably
|
||||
@@ -113,7 +109,6 @@ jobs:
|
||||
# continue-on-error: true at step level — job-level is ignored by Gitea
|
||||
# Actions (quirk #10, internal runbooks). Belt-and-suspenders with
|
||||
# SOP_FAIL_OPEN=1 + || true below.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# sop-tier-refire — manual fallback for sop-tier-check refire.
|
||||
# sop-tier-refire — issue_comment-triggered refire of sop-tier-check.
|
||||
#
|
||||
# Closes internal#292. Gitea 1.22.6 doesn't refire workflows on the
|
||||
# `pull_request_review` event (go-gitea/gitea#33700); the `sop-tier-check`
|
||||
@@ -8,12 +8,12 @@
|
||||
# to merge is the admin force-merge path (audited via `audit-force-merge`
|
||||
# but the audit trail keeps growing; see `feedback_never_admin_merge_bypass`).
|
||||
#
|
||||
# Comment-triggered refires now live in `review-refire-comments.yml`. Gitea
|
||||
# queues issue_comment workflows before evaluating job-level `if:`, so having
|
||||
# qa-review, security-review, sop-checklist, and sop-tier-refire all subscribe
|
||||
# to every comment caused queue storms on SOP-heavy PRs. This workflow is a
|
||||
# non-automatic breadcrumb only; Gitea 1.22.6 does not support
|
||||
# workflow_dispatch inputs, so real refires must use `/refire-tier-check`.
|
||||
# Workaround pattern from `feedback_pull_request_review_no_refire`:
|
||||
# `issue_comment` events DO fire reliably on 1.22.6. When a repo
|
||||
# MEMBER/OWNER/COLLABORATOR comments `/refire-tier-check` on a PR, this
|
||||
# workflow re-runs the sop-tier-check logic and POSTs the resulting
|
||||
# status to the PR head SHA directly. No empty commit, no git history
|
||||
# bloat, no cascade re-fire of every other workflow on the PR.
|
||||
#
|
||||
# SECURITY MODEL:
|
||||
#
|
||||
@@ -37,16 +37,43 @@
|
||||
# Rate-limit: a 1s pre-sleep + a "skip if status posted in last 30s"
|
||||
# guard prevents comment-spam from thrashing the status. See the script.
|
||||
|
||||
name: sop-tier-check refire (manual)
|
||||
name: sop-tier-check refire (issue_comment)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
refire:
|
||||
# Three gates, all required:
|
||||
# - comment is on a PR (not a plain issue)
|
||||
# - commenter is MEMBER, OWNER, or COLLABORATOR
|
||||
# - comment body contains the slash-command trigger
|
||||
if: |
|
||||
github.event.issue.pull_request != null &&
|
||||
contains(fromJson('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) &&
|
||||
contains(github.event.comment.body, '/refire-tier-check')
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
steps:
|
||||
- name: Explain supported refire path
|
||||
run: |
|
||||
echo "::error::Gitea 1.22.6 does not support workflow_dispatch inputs here; comment /refire-tier-check on the PR instead."
|
||||
exit 1
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Load the script from the default branch (main), matching the
|
||||
# sop-tier-check.yml security model.
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
- name: Re-evaluate sop-tier-check and POST status
|
||||
env:
|
||||
# Same org-level secret sop-tier-check.yml + audit-force-merge.yml use.
|
||||
# Fallback to GITHUB_TOKEN with a clear error if missing.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
|
||||
# Set to '1' for diagnostic per-API-call output. Off by default.
|
||||
SOP_DEBUG: '0'
|
||||
run: bash .gitea/scripts/sop-tier-refire.sh
|
||||
|
||||
@@ -81,11 +81,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the smoke's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -112,9 +107,9 @@ jobs:
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the smoke to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: MiniMax-M2
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "smoke-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the smoke script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
@@ -134,12 +129,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
|
||||
@@ -75,22 +75,16 @@ permissions:
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
# SSOT-Instance-10 (#333): triplet sourced from org/repo var `ECR_REGISTRY` with
|
||||
# the current prod-account literal as bootstrap fallback. When the org var is set,
|
||||
# the fallback becomes dead code and switching accounts/regions is a one-line
|
||||
# change at the org level. Pattern mirrors `vars.CP_URL || 'literal'` below.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: post-merge staging verification side effect; CI / all-required gates merges.
|
||||
staging-smoke:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
sha: ${{ steps.compute.outputs.sha }}
|
||||
@@ -195,7 +189,6 @@ jobs:
|
||||
echo "assertions in the staging-smoke step log above."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# bp-exempt: post-merge image promotion side effect; staging-smoke controls promotion.
|
||||
promote-to-latest:
|
||||
# On green, calls the CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha> to promote the verified ECR image. This is the same
|
||||
@@ -212,7 +205,6 @@ jobs:
|
||||
if: ${{ needs.staging-smoke.result == 'success' && needs.staging-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
env:
|
||||
SHA: ${{ needs.staging-smoke.outputs.sha }}
|
||||
@@ -239,11 +231,6 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_TAG="staging-${SHA}"
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. This caller promotes
|
||||
# the verified staging image across the entire prod fleet (canary
|
||||
# + fan-out), no slug scoping, so confirm:true is correct.
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--argjson soak "${SOAK_SECONDS:-120}" \
|
||||
@@ -253,8 +240,7 @@ jobs:
|
||||
target_tag: $tag,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry,
|
||||
confirm: true
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
if [ -n "${CANARY_SLUG:-}" ]; then
|
||||
|
||||
@@ -53,12 +53,19 @@ name: status-reaper
|
||||
# `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as
|
||||
# "unknown on type" when `workflow_dispatch.inputs.X` is present.
|
||||
on:
|
||||
# Schedule moved to operator-config:
|
||||
# /etc/cron.d/molecule-core-status-reaper ->
|
||||
# /usr/local/bin/molecule-core-cron-bot.sh status-reaper
|
||||
#
|
||||
# This keeps the 5-minute compensation cadence but stops a maintenance
|
||||
# bot from consuming Gitea Actions runner slots during PR merge waves.
|
||||
# SCHEDULE RE-ENABLED 2026-05-12 rev3 — interim disable (mc#645) reverted now that
|
||||
# rev3 widens DEFAULT_SWEEP_LIMIT 10 → 30 (covers retroactive-failure timing window).
|
||||
# Sibling watchdog re-enabled in the same PR with timeout-minutes raised 5 → 15.
|
||||
schedule:
|
||||
# Every 5 minutes. Off-zero alignment with sibling cron workflows:
|
||||
# ci-required-drift (`:17`), main-red-watchdog (`:05`),
|
||||
# railway-pin-audit (`:23`). 5-min cadence gives a tight enough
|
||||
# close on schedule-triggered false-reds that main-red-watchdog
|
||||
# (hourly :05) almost never files an issue on the false case.
|
||||
# rev3 keeps `*/5` unchanged per hongming-pc2 03:25Z review:
|
||||
# "trades window-width-cheap for cadence-loady" — N=30 widens
|
||||
# the lookback cheaply without doubling runner load via `*/2`.
|
||||
- cron: '*/5 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
# Compensating-status POST needs write on repo statuses; no other
|
||||
@@ -77,7 +84,7 @@ permissions:
|
||||
jobs:
|
||||
reap:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
timeout-minutes: 3
|
||||
steps:
|
||||
- name: Check out repo at default-branch HEAD
|
||||
# BASE checkout per `feedback_pull_request_target_workflow_from_base`.
|
||||
@@ -111,7 +118,4 @@ jobs:
|
||||
REPO: ${{ github.repository }}
|
||||
WATCH_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
STATUS_REAPER_API_RETRIES: "4"
|
||||
STATUS_REAPER_API_TIMEOUT_SEC: "20"
|
||||
STATUS_REAPER_API_RETRY_SLEEP_SEC: "2"
|
||||
run: python3 .gitea/scripts/status-reaper.py
|
||||
|
||||
@@ -29,11 +29,15 @@ name: Sweep stale AWS Secrets Manager secrets
|
||||
# reconciler enumerator) is filed as a separate controlplane
|
||||
# issue. This sweeper is the immediate cost-relief stopgap.
|
||||
#
|
||||
# AWS credentials: use the dedicated Secrets Manager janitor principal.
|
||||
# Do not fall back to the molecule-cp application principal: it does
|
||||
# not need account-wide ListSecrets, and a 2026-05-12 CI failure proved
|
||||
# that using it here turns a least-privilege production credential into
|
||||
# a red scheduled janitor.
|
||||
# AWS credentials: the confirmed Gitea secrets are AWS_ACCESS_KEY_ID /
|
||||
# AWS_SECRET_ACCESS_KEY (the molecule-cp IAM user). These are the same
|
||||
# credentials used by the rest of the platform. The dedicated
|
||||
# AWS_JANITOR_* naming (which the original GitHub workflow used) was
|
||||
# never populated in Gitea — the existing secrets are AWS_ACCESS_KEY_ID /
|
||||
# AWS_SECRET_ACCESS_KEY (per issue #425 §425 audit). These DO have
|
||||
# secretsmanager:ListSecrets (the production molecule-cp principal);
|
||||
# if ListSecrets is revoked in future, a dedicated janitor principal
|
||||
# would need to be created and the Gitea secret names updated here.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
|
||||
# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
|
||||
@@ -41,12 +45,10 @@ name: Sweep stale AWS Secrets Manager secrets
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :30, offset from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45). This janitor is intentionally schedule-only
|
||||
# for deletes; manual dispatch is forced to dry-run below because Gitea
|
||||
# 1.22.6 rejects workflow_dispatch.inputs.
|
||||
# Hourly at :30 — offsets from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45) so the three janitors don't burst the
|
||||
# CP admin endpoints at the same minute.
|
||||
- cron: '30 * * * *'
|
||||
workflow_dispatch:
|
||||
# Don't let two sweeps race the same AWS account.
|
||||
concurrency:
|
||||
group: sweep-aws-secrets
|
||||
@@ -62,24 +64,21 @@ jobs:
|
||||
sweep:
|
||||
name: Sweep AWS Secrets Manager
|
||||
runs-on: ubuntu-latest
|
||||
# This is a cost/leak janitor. A scheduled failure must be red so
|
||||
# operators know tenant bootstrap secrets may be leaking.
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 30 min cap, mirroring the other janitors. AWS DeleteSecret is
|
||||
# fast (~0.3s/call) so even a 100+ backlog drains in seconds
|
||||
# under the 8-way xargs parallelism, but the cap is set generously
|
||||
# to leave headroom for any actual API hang.
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# Keep this literal. Gitea/act_runner 1.22.6 can mis-render
|
||||
# secret-backed expressions with `||`, which produced an invalid
|
||||
# Secrets Manager endpoint in the scheduled janitor.
|
||||
AWS_REGION: us-east-2
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_SECRETS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_DELETE_PCT: 50
|
||||
GRACE_HOURS: 24
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -114,25 +113,17 @@ jobs:
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry:
|
||||
# - schedule: execute (the whole point of an hourly janitor).
|
||||
# - workflow_dispatch: dry-run. Gitea 1.22.6 rejects
|
||||
# workflow_dispatch.inputs, so there is no safe manual
|
||||
# "flip it to execute" toggle in this workflow.
|
||||
# The script's MAX_DELETE_PCT gate (default 50%) remains the
|
||||
# second line of defense regardless of trigger.
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-aws-secrets.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-aws-secrets.sh --execute
|
||||
fi
|
||||
|
||||
- name: Notify on sweep failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "::error::sweep-aws-secrets FAILED — AWS tenant bootstrap secrets may be leaking. Check missing Gitea secrets, staging/prod CP admin tokens, AWS janitor IAM permissions, or the script safety gate."
|
||||
exit 1
|
||||
|
||||
@@ -71,7 +71,6 @@ jobs:
|
||||
name: Sweep CF orphans
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
# 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
|
||||
# within one cron interval instead of burning a full tick. Realistic
|
||||
|
||||
@@ -55,7 +55,6 @@ jobs:
|
||||
name: Sweep CF tunnels
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
# 30 min cap. Was 5 min on the theory that the only thing that
|
||||
# could take >5min is a CF-API hang — but on 2026-05-02 a backlog
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
name: sync-providers-yaml
|
||||
|
||||
# Cross-repo canonical↔synced-copy drift gate (internal#718 P2-A, CTO
|
||||
# 2026-05-27 "Distribution = SDK via codegen + verify-CI", multi-repo branch:
|
||||
# "codegen-checked-into-each-repo + verify-CI").
|
||||
#
|
||||
# The canonical provider-registry SSOT is molecule-controlplane
|
||||
# internal/providers/providers.yaml. molecule-core has NO Go module dependency
|
||||
# on controlplane, so instead of importing it we carry a SYNCED COPY at
|
||||
# workspace-server/internal/providers/providers.yaml and gate it.
|
||||
#
|
||||
# This workflow fetches the canonical providers.yaml from controlplane (via the
|
||||
# Gitea raw endpoint, read-only) and byte-compares it against core's synced
|
||||
# copy. RED if they differ — meaning the canonical moved and core's copy must be
|
||||
# re-synced (copy verbatim + `go generate ./...` + bump
|
||||
# canonicalProvidersYAMLSHA256 in sync_canonical_test.go).
|
||||
#
|
||||
# Pairs with:
|
||||
# * sync_canonical_test.go — hermetic sha pin (catches a hand-edit of core's
|
||||
# copy even with no network); runs in the normal `go test ./...`.
|
||||
# * verify-providers-gen.yml — artifact ↔ synced-copy drift.
|
||||
#
|
||||
# ENFORCEMENT GATING: standalone workflow, NOT a job in ci.yml and NOT in
|
||||
# branch protection (same soak-then-promote posture as verify-providers-gen).
|
||||
# It is intentionally absent from ci.yml's job set so the ci-required-drift
|
||||
# sentinel does not fire on it.
|
||||
#
|
||||
# AUTH: uses AUTO_SYNC_TOKEN (the existing cross-repo read token used to sync
|
||||
# template/provider content from sibling repos). If the secret is absent the
|
||||
# job emits a clear ::warning:: and exits 0 — the hermetic sha pin in
|
||||
# sync_canonical_test.go is the always-on backstop, so a missing cross-repo
|
||||
# token degrades to "hand-edit still caught, live canonical drift not caught"
|
||||
# rather than a hard red that blocks unrelated PRs.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- 'workspace-server/internal/providers/providers.yaml'
|
||||
- '.gitea/workflows/sync-providers-yaml.yml'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/internal/providers/providers.yaml'
|
||||
- '.gitea/workflows/sync-providers-yaml.yml'
|
||||
schedule:
|
||||
# Daily at :23 — catch a canonical change in controlplane that landed
|
||||
# without a paired core re-sync PR (off-zero to spread cron load).
|
||||
- cron: '23 4 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: sync-providers-yaml-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
compare:
|
||||
name: Compare synced providers.yaml against controlplane canonical
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 6
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Fetch canonical providers.yaml from controlplane and byte-compare
|
||||
env:
|
||||
AUTO_SYNC_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
API_ROOT: ${{ github.server_url }}/api/v1
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${AUTO_SYNC_TOKEN:-}" ]; then
|
||||
echo "::warning::AUTO_SYNC_TOKEN secret missing — skipping the live cross-repo compare."
|
||||
echo "The hermetic sha pin (sync_canonical_test.go) still gates hand-edits of core's copy."
|
||||
echo "Provision AUTO_SYNC_TOKEN (read scope on molecule-controlplane) to enable live canonical-drift detection."
|
||||
exit 0
|
||||
fi
|
||||
CANON_URL="${API_ROOT}/repos/molecule-ai/molecule-controlplane/raw/internal/providers/providers.yaml?ref=main"
|
||||
# Use the /raw endpoint: it returns the file bytes directly. (The
|
||||
# /contents endpoint ignores Accept: application/vnd.gitea.raw on
|
||||
# Gitea 1.22.6 and returns the JSON+base64 envelope, which made this
|
||||
# diff a permanent false RED.)
|
||||
curl -fsS \
|
||||
-H "Authorization: token ${AUTO_SYNC_TOKEN}" \
|
||||
"${CANON_URL}" -o /tmp/canonical-providers.yaml
|
||||
LOCAL=workspace-server/internal/providers/providers.yaml
|
||||
if diff -u /tmp/canonical-providers.yaml "$LOCAL"; then
|
||||
echo "OK — core's synced providers.yaml is byte-identical to the controlplane canonical."
|
||||
else
|
||||
echo "::error::core's synced providers.yaml DRIFTED from the controlplane canonical (SSOT)."
|
||||
echo "Re-sync: copy controlplane internal/providers/providers.yaml verbatim over"
|
||||
echo " $LOCAL, run 'go generate ./...' in workspace-server/, and bump"
|
||||
echo " canonicalProvidersYAMLSHA256 in internal/providers/sync_canonical_test.go."
|
||||
exit 1
|
||||
fi
|
||||
@@ -11,9 +11,8 @@ name: Ops Scripts Tests
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Runs the unittest suite for scripts/ on every PR + push that touches
|
||||
# anything under scripts/ or .gitea/scripts/. Kept separate from the main CI
|
||||
# so a script-only change doesn't trigger the heavier Go/Canvas/Python
|
||||
# pipelines.
|
||||
# anything under scripts/. Kept separate from the main CI so a script-only
|
||||
# change doesn't trigger the heavier Go/Canvas/Python pipelines.
|
||||
#
|
||||
# Discovery layout: tests sit alongside the code they test (see
|
||||
# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
|
||||
@@ -28,13 +27,11 @@ on:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
|
||||
env:
|
||||
@@ -49,31 +46,20 @@ jobs:
|
||||
name: Ops scripts (unittest)
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Install .gitea script test dependencies
|
||||
run: python -m pip install --quiet 'pytest==9.0.2' 'PyYAML==6.0.2'
|
||||
- name: Run scripts/ unittests, if any
|
||||
# Top-level scripts/ tests live alongside their target file. The
|
||||
# runtime packaging tests moved to molecule-ai-workspace-runtime, so
|
||||
# this pass may legitimately find no tests.
|
||||
- name: Run scripts/ unittests (build_runtime_package, ...)
|
||||
# Top-level scripts/ tests live alongside their target file
|
||||
# (e.g. scripts/test_build_runtime_package.py exercises
|
||||
# scripts/build_runtime_package.py). discover from scripts/
|
||||
# picks up only top-level test_*.py because scripts/ops/ has
|
||||
# no __init__.py — that's intentional, so we run two passes.
|
||||
working-directory: scripts
|
||||
run: |
|
||||
set +e
|
||||
python -m unittest discover -t . -p 'test_*.py' -v
|
||||
rc=$?
|
||||
if [ "$rc" -eq 5 ]; then
|
||||
echo "No top-level scripts/ unittest files found; skipping."
|
||||
exit 0
|
||||
fi
|
||||
exit "$rc"
|
||||
run: python -m unittest discover -t . -p 'test_*.py' -v
|
||||
- name: Run scripts/ops/ unittests (sweep_cf_decide, ...)
|
||||
working-directory: scripts/ops
|
||||
run: python -m unittest discover -p 'test_*.py' -v
|
||||
- name: Run .gitea/scripts pytest suite
|
||||
run: python -m pytest .gitea/scripts/tests -q
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
name: verify-providers-gen
|
||||
|
||||
# Provider-registry SSOT enforcement gate — molecule-core side (internal#718
|
||||
# P2-A, CTO 2026-05-27 "Distribution = SDK via codegen + verify-CI").
|
||||
#
|
||||
# The canonical schema SSOT is molecule-controlplane
|
||||
# internal/providers/providers.yaml. molecule-core carries a SYNCED COPY at
|
||||
# workspace-server/internal/providers/providers.yaml (kept in sync by the
|
||||
# companion sync-providers-yaml.yml gate), and cmd/gen-providers emits the
|
||||
# checked-in Go projection workspace-server/internal/providers/gen/registry_gen.go.
|
||||
#
|
||||
# This workflow regenerates the artifact into the working tree and fails RED if
|
||||
# it differs from what is committed — catching BOTH:
|
||||
# * a providers.yaml (synced-copy) change that wasn't followed by `go generate ./...`, and
|
||||
# * a hand-edit of the generated artifact (it carries a DO NOT EDIT header).
|
||||
#
|
||||
# It is the molecule-core mirror of molecule-controlplane's verify-providers-gen
|
||||
# workflow. Together with sync-providers-yaml (canonical↔synced-copy drift) it
|
||||
# closes the codegen-checked-into-each-repo + verify-CI loop the RFC mandates.
|
||||
#
|
||||
# ENFORCEMENT GATING (deliberate, per dev-SOP "implementation gating"):
|
||||
# this is a STANDALONE workflow, NOT a job inside ci.yml, and is NOT yet in any
|
||||
# branch-protection status_check_contexts. Rationale (identical to the CP P0
|
||||
# rollout):
|
||||
# * It runs + reports RED on every PR/push immediately (visible signal).
|
||||
# * It is intentionally absent from ci.yml's job set so the ci-required-drift
|
||||
# sentinel (jobs ↔ branch-protection ↔ audit-env) does NOT fire on it, and
|
||||
# from branch protection (turning it into a hard merge gate has blast radius
|
||||
# — operator GO required, same pattern as sop-tier-check / verify-providers-gen
|
||||
# on controlplane). Promote it into branch protection in a follow-up once
|
||||
# P2 has soaked.
|
||||
# Until then it behaves like secret-scan / block-internal-paths: a standalone
|
||||
# advisory-to-hard gate the author is expected to keep green.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
# CI-scheduler-overload fix (fix/ci-scheduler-fanout, 2026-06-01):
|
||||
# this gate only verifies that the generated providers artifact is in
|
||||
# sync with the schema SSOT. Its verdict can ONLY change when one of
|
||||
# the codegen inputs/outputs changes, so firing the Go toolchain on
|
||||
# every unrelated PR (docs, canvas, scripts) is pure fan-out cost.
|
||||
# Scoped to the codegen surface. SAFE because this workflow is NOT a
|
||||
# branch-protection status_check_context (see header §ENFORCEMENT
|
||||
# GATING) — lint-required-no-paths only forbids paths filters on
|
||||
# REQUIRED workflows; this is advisory, so a paths filter is allowed.
|
||||
# Mirrors the sibling sync-providers-yaml.yml scoping convention.
|
||||
paths:
|
||||
- 'workspace-server/internal/providers/**'
|
||||
- 'workspace-server/cmd/gen-providers/**'
|
||||
- '.gitea/workflows/verify-providers-gen.yml'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/internal/providers/**'
|
||||
- 'workspace-server/cmd/gen-providers/**'
|
||||
- '.gitea/workflows/verify-providers-gen.yml'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: verify-providers-gen-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
name: Regenerate providers artifact and fail on drift
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
|
||||
- name: Verify generated artifact is in sync with providers.yaml
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# -check regenerates in memory and byte-compares against the
|
||||
# checked-in artifact; exit 1 (RED) on any drift. This is the
|
||||
# single source of the gate's verdict — the same code path
|
||||
# `go test ./cmd/gen-providers` exercises.
|
||||
go run ./cmd/gen-providers -check
|
||||
|
||||
- name: Belt-and-braces — regenerate in place and assert clean tree
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Independent confirmation that does not trust the -check path:
|
||||
# actually write the artifact and assert git sees no change. If
|
||||
# this and the step above ever disagree, the gate is suspect.
|
||||
go generate ./...
|
||||
if ! git diff --quiet -- internal/providers/gen/registry_gen.go; then
|
||||
echo "::error::workspace-server/internal/providers/gen/registry_gen.go drifted from providers.yaml."
|
||||
echo "Run 'go generate ./...' (or 'go run ./cmd/gen-providers') in workspace-server/ and commit the result."
|
||||
git --no-pager diff -- internal/providers/gen/registry_gen.go | head -80
|
||||
exit 1
|
||||
fi
|
||||
echo "OK — generated providers artifact is in sync with the schema SSOT."
|
||||
@@ -31,7 +31,6 @@ jobs:
|
||||
name: Weekly Platform-Go Surface
|
||||
runs-on: ubuntu-latest
|
||||
# continue-on-error: surface only, never block
|
||||
# mc#1982: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
@@ -106,7 +105,7 @@ jobs:
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !(\$pct < 10)}" || continue
|
||||
rel=$(echo "$file" | sed 's|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/workspace-server/||; s|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/||')
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
name: Block internal-flavored paths
|
||||
|
||||
# Hard CI gate. Internal content (positioning, competitive briefs, sales
|
||||
# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
|
||||
# this public monorepo must never re-acquire those paths. CEO directive
|
||||
# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
|
||||
#
|
||||
# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
|
||||
# briefs into the easiest path their cwd resolves to (root /research,
|
||||
# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
|
||||
# or a stale gitignore line. This workflow is the mechanical backstop.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
# Required for GitHub merge queue: the queue's pre-merge CI run on
|
||||
# `gh-readonly-queue/...` refs needs this check to fire so the queue
|
||||
# gets a real result instead of stalling forever AWAITING_CHECKS.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Block forbidden paths
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base is github.event.pull_request.base.sha,
|
||||
# which may be many commits behind HEAD and therefore absent from the
|
||||
# shallow clone above. Fetch it explicitly (depth=1 keeps it fast).
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
# For merge_group events the queue's pre-merge ref is a commit on
|
||||
# `gh-readonly-queue/...` whose parent is the queue's base_sha.
|
||||
# That parent isn't part of the queue branch's shallow clone, so
|
||||
# we fetch it explicitly. Mirrors the equivalent step in
|
||||
# secret-scan.yml (#2120) — same shallow-clone bug class.
|
||||
- name: Fetch merge_group base SHA (merge_group events only)
|
||||
if: github.event_name == 'merge_group'
|
||||
run: git fetch --depth=1 origin ${{ github.event.merge_group.base_sha }}
|
||||
|
||||
- name: Refuse if forbidden paths appear
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# merge_group has its own base_sha/head_sha; pull_request has
|
||||
# pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
MG_BASE_SHA: ${{ github.event.merge_group.base_sha }}
|
||||
MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Paths that must NEVER live in the public monorepo. Add to this
|
||||
# list narrowly — broader patterns belong in .gitignore so day-to-day
|
||||
# docs work isn't accidentally blocked.
|
||||
FORBIDDEN_PATTERNS=(
|
||||
"^research/"
|
||||
"^marketing/"
|
||||
"^docs/marketing/"
|
||||
"^comment-[0-9]+\.json$"
|
||||
"^test-pmm.*\.(txt|md)$"
|
||||
"^tick-reflections.*\.(txt|md)$"
|
||||
".*-temp\.(md|txt)$"
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
merge_group)
|
||||
BASE="$MG_BASE_SHA"
|
||||
HEAD="$MG_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
# Same recovery shape as secret-scan.yml (#2120 — incident
|
||||
# 2026-04-27 06:50Z block-internal-paths exit 128 with
|
||||
# "fatal: bad object <sha>" on staging push).
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check
|
||||
# the entire tree as if every file were new. Slower but
|
||||
# correct on first push or post-fetch-failure recovery.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OFFENDING=""
|
||||
for path in $CHANGED; do
|
||||
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
|
||||
if echo "$path" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Forbidden internal-flavored paths detected:"
|
||||
printf "$OFFENDING"
|
||||
echo ""
|
||||
echo "These paths belong in molecule-ai/internal, not this public repo."
|
||||
echo "See docs/internal-content-policy.md for canonical locations."
|
||||
echo ""
|
||||
echo "If your file is genuinely public-facing (e.g. a blog post"
|
||||
echo "ready to ship), use one of these alternatives instead:"
|
||||
echo " • Public-bound blog posts: docs/blog/<slug>.md"
|
||||
echo " • Public-bound tutorials: docs/tutorials/<slug>.md"
|
||||
echo " • Public devrel content: docs/devrel/<slug>.md"
|
||||
echo ""
|
||||
echo "If you legitimately need to add a new top-level path that"
|
||||
echo "happens to match a forbidden pattern, edit"
|
||||
echo ".github/workflows/block-internal-paths.yml and update the"
|
||||
echo "FORBIDDEN_PATTERNS list with reviewer signoff."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No forbidden paths in this change."
|
||||
@@ -0,0 +1,320 @@
|
||||
name: Canary — staging SaaS smoke (every 30 min)
|
||||
|
||||
# Minimum viable health check: provisions one Hermes workspace on a fresh
|
||||
# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
|
||||
# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
|
||||
# issue on the next green run.
|
||||
#
|
||||
# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
|
||||
# but runs only on provisioning-critical pushes + nightly — this one
|
||||
# catches drift in the 30-min window between those runs (AMI health, CF
|
||||
# cert rotation, WorkOS session stability, etc.).
|
||||
#
|
||||
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
|
||||
# peers/activity checks. One parent workspace + one A2A turn is enough
|
||||
# to signal "SaaS stack end-to-end is alive."
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 min. Cron on GitHub-hosted runners has a known drift of
|
||||
# a few minutes under load — that's fine for a canary.
|
||||
- cron: '*/30 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
keep_on_failure:
|
||||
description: >-
|
||||
Skip teardown when the canary fails (debugging only). The
|
||||
tenant org + EC2 + CF tunnel + DNS stay alive so an operator
|
||||
can SSM into the workspace EC2 and capture docker logs of the
|
||||
failing claude-code container. REMEMBER to manually delete
|
||||
via DELETE /cp/admin/tenants/<slug> when done so the org
|
||||
doesn't accumulate cost. Only honored on workflow_dispatch;
|
||||
cron runs always tear down (we don't want unattended cron
|
||||
to leak resources).
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# Serialise with the full-SaaS workflow so they don't contend for the
|
||||
# same org-create quota on staging. Different group key from
|
||||
# e2e-staging-saas since we don't mind queueing canaries behind one
|
||||
# full run, but two canaries SHOULD queue against each other.
|
||||
concurrency:
|
||||
group: canary-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
# Needed to open / close the alerting issue.
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
canary:
|
||||
name: Canary smoke
|
||||
runs-on: ubuntu-latest
|
||||
# 25 min headroom over the 15-min TLS-readiness deadline in
|
||||
# tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
|
||||
# the job is killed at the wall-clock 15:00 mark BEFORE the bash
|
||||
# `fail` + diagnostic burst can fire, leaving every cancellation
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping
|
||||
# canary tighter than them so a true wedge still surfaces here
|
||||
# first.
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the canary red the entire time). claude-code template's
|
||||
# `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
|
||||
# ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
|
||||
# billing account, so OpenAI quota collapse no longer wedges the
|
||||
# canary. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
|
||||
# full_saas.sh branches SECRETS_JSON on which key is present —
|
||||
# MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes overridden via workflow_dispatch can still
|
||||
# exercise the OpenAI path without re-editing the workflow.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "canary-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the canary script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# tenant org + EC2 stay alive for SSM-based log capture. Cron runs
|
||||
# never set this (the input only exists on workflow_dispatch) so
|
||||
# unattended cron always tears down. See molecule-core#129
|
||||
# failure mode #1 — capturing the actual exception requires
|
||||
# docker logs from the live container.
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per the lesson from synth E2E #2578:
|
||||
# an empty key silently falls through to the wrong
|
||||
# SECRETS_JSON branch and the canary fails 5 min later with
|
||||
# a confusing auth error instead of the clean "secret
|
||||
# missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain. Operators only need to set ONE of these
|
||||
# secrets; we don't force a choice between them.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: Canary run
|
||||
id: canary
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Alerting: open a sticky issue on the FIRST failure; comment on
|
||||
# subsequent failures; auto-close on next green. Comment-on-existing
|
||||
# de-duplicates so a single open issue accumulates the streak —
|
||||
# ops sees one issue with N comments rather than N issues.
|
||||
#
|
||||
# Why no consecutive-failures threshold (e.g., wait 3 runs before
|
||||
# filing): the prior threshold check used
|
||||
# `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
|
||||
# not expose (returns 404). On Gitea Actions the threshold call
|
||||
# ALWAYS failed, breaking the entire alerting step and going days
|
||||
# silent on real regressions (38h+ chronic red on 2026-05-07/08
|
||||
# before this fix; tracked in molecule-core#129). Filing on first
|
||||
# failure is also better UX — we want to know about the first red,
|
||||
# not wait 90 min for it to "count." Real flakes get one issue +
|
||||
# a quick close-on-green; persistent reds accumulate comments.
|
||||
- name: Open issue on failure
|
||||
if: failure()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const runURL = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
// Find an existing open canary issue (stable title match).
|
||||
// If one exists, this isn't a "first failure" — comment and exit.
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary still failing. ${runURL}`,
|
||||
});
|
||||
core.info(`Commented on existing issue #${match.number}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// No open issue yet — file one on this first failure. The
|
||||
// comment-on-existing branch above means subsequent failures
|
||||
// accumulate as comments on this same issue, so we don't
|
||||
// spam new issues per run.
|
||||
const body =
|
||||
`Canary run failed at ${new Date().toISOString()}.\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`This issue auto-closes on the next green canary run. ` +
|
||||
`Consecutive failures add a comment here rather than a new issue.`;
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['canary-staging', 'bug'],
|
||||
});
|
||||
core.info('Opened canary failure issue (first red)');
|
||||
|
||||
- name: Auto-close canary issue on success
|
||||
if: success()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const { data: open } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = open.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary recovered at ${new Date().toISOString()}. Closing.`,
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
state: 'closed',
|
||||
});
|
||||
core.info(`Closed recovered canary issue #${match.number}`);
|
||||
}
|
||||
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
# Slug prefix matches what test_staging_full_saas.sh emits
|
||||
# in canary mode:
|
||||
# SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier this was `e2e-{today}-canary-` — that was the
|
||||
# full-mode pattern (date FIRST, mode SECOND); canary slugs
|
||||
# have mode FIRST, date SECOND. The mismatch silently
|
||||
# never matched, leaving every cancelled-canary EC2 alive
|
||||
# until the once-an-hour sweep eventually caught it
|
||||
# (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
|
||||
# cleanup; same gap on three earlier cancellations today).
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope to slugs from THIS canary run when GITHUB_RUN_ID is
|
||||
# available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
|
||||
# so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
|
||||
# full-mode safety net's per-run scoping (e2e-staging-saas.yml)
|
||||
# added after the 2026-04-21 cross-run cleanup incident.
|
||||
# Sweep both today AND yesterday's UTC dates so a run that
|
||||
# crosses midnight still cleans up its own slug — see the
|
||||
# 2026-04-26→27 canvas-safety-net incident.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug DELETE with HTTP-code verification. The previous
|
||||
# `... >/dev/null || true` swallowed every failure, so a 5xx
|
||||
# or timeout from CP looked identical to "successfully cleaned
|
||||
# up" and the tenant kept eating ~2 vCPU until the hourly
|
||||
# stale sweep caught it (up to 2h later). Now we capture the
|
||||
# response code and surface non-2xx as a workflow warning, so
|
||||
# the run page shows which slug leaked. We still don't `exit 1`
|
||||
# on cleanup failure — a single-canary cleanup miss shouldn't
|
||||
# fail-flag the canary itself when the actual smoke check
|
||||
# passed. The sweep-stale-e2e-orgs cron (now every 15 min,
|
||||
# 30-min threshold) is the safety net for whatever slips past.
|
||||
# See molecule-controlplane#420.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,255 @@
|
||||
name: canary-verify
|
||||
|
||||
# Runs the canary smoke suite against the staging canary tenant fleet
|
||||
# after a new :staging-<sha> image lands in ECR. On green, calls the
|
||||
# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
|
||||
# the prod tenant fleet's 5-minute auto-updater picks up the verified
|
||||
# digest. On red, :latest stays on the prior known-good digest and
|
||||
# prod is untouched.
|
||||
#
|
||||
# Registry note (2026-05-10): This workflow previously used GHCR
|
||||
# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
|
||||
# during the 2026-05-06 Gitea suspension migration when publish-
|
||||
# workspace-server-image.yml switched to the operator's ECR org
|
||||
# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
|
||||
# platform-tenant). The GHCR → ECR migration was never applied to
|
||||
# this file, so canary-verify was silently smoke-testing the stale
|
||||
# GHCR image while the actual staging/prod tenants ran the ECR image.
|
||||
# Result: smoke tests could not catch a broken ECR build. Fix:
|
||||
# - Wait step: reads SHA from running canary /health (tenant-
|
||||
# agnostic, works regardless of registry).
|
||||
# - Promote step: calls CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
|
||||
# No longer attempts GHCR crane ops.
|
||||
#
|
||||
# Dependencies:
|
||||
# - publish-workspace-server-image.yml publishes :staging-<sha>
|
||||
# to ECR on staging and main merges.
|
||||
# - Canary tenants are configured to pull :staging-<sha> from ECR
|
||||
# (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
|
||||
# - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
|
||||
# CANARY_CP_SHARED_SECRET are populated.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["publish-workspace-server-image"]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
|
||||
jobs:
|
||||
canary-smoke:
|
||||
# Skip when the upstream workflow failed — no image to test against.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
sha: ${{ steps.compute.outputs.sha }}
|
||||
smoke_ran: ${{ steps.smoke.outputs.ran }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Compute sha
|
||||
id: compute
|
||||
run: echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Wait for canary tenants to pick up :staging-<sha>
|
||||
# Poll canary health endpoints every 30s for up to 7 min instead
|
||||
# of a fixed 6-min sleep. Exits as soon as ALL canaries report
|
||||
# the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
|
||||
# proceeding after 7 min even if not all canaries responded —
|
||||
# the smoke suite will catch any that didn't update.
|
||||
#
|
||||
# NOTE: The SHA is read from the running tenant's /health response,
|
||||
# NOT from a registry lookup. This is registry-agnostic and works
|
||||
# regardless of whether the tenant pulls from ECR, GHCR, or any
|
||||
# other registry — the canary is telling us what it's actually
|
||||
# running, which is the ground truth for smoke testing.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
|
||||
run: |
|
||||
if [ -z "$CANARY_TENANT_URLS" ]; then
|
||||
echo "No canary URLs configured — falling back to 60s wait"
|
||||
sleep 60
|
||||
exit 0
|
||||
fi
|
||||
IFS=',' read -ra URLS <<< "$CANARY_TENANT_URLS"
|
||||
MAX_WAIT=420 # 7 minutes
|
||||
INTERVAL=30
|
||||
ELAPSED=0
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
ALL_READY=true
|
||||
for url in "${URLS[@]}"; do
|
||||
HEALTH=$(curl -s --max-time 5 "${url}/health" 2>/dev/null || echo "{}")
|
||||
SHA=$(echo "$HEALTH" | grep -o "\"sha\":\"[^\"]*\"" | head -1 | cut -d'"' -f4)
|
||||
if [ "$SHA" != "$EXPECTED_SHA" ]; then
|
||||
ALL_READY=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
if $ALL_READY; then
|
||||
echo "All canaries running staging-${EXPECTED_SHA} after ${ELAPSED}s"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for canaries... (${ELAPSED}s / ${MAX_WAIT}s)"
|
||||
sleep $INTERVAL
|
||||
ELAPSED=$((ELAPSED + INTERVAL))
|
||||
done
|
||||
echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
|
||||
|
||||
- name: Run canary smoke suite
|
||||
id: smoke
|
||||
# Graceful-skip when no canary fleet is configured (Phase 2 not yet
|
||||
# stood up — see molecule-controlplane/docs/canary-tenants.md).
|
||||
# Sets `ran=false` on skip so promote-to-latest stays off (we don't
|
||||
# want every main merge auto-promoting without gating). Manual
|
||||
# promote-latest.yml is the release gate while canary is absent.
|
||||
# Once the fleet is real: delete the early-exit branch.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
|
||||
CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
|
||||
CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${CANARY_TENANT_URLS:-}" ] \
|
||||
|| [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
|
||||
|| [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
|
||||
{
|
||||
echo "## ⚠️ canary-verify skipped"
|
||||
echo
|
||||
echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
|
||||
echo "Phase 2 canary fleet has not been stood up yet —"
|
||||
echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
|
||||
echo
|
||||
echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "ran=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::canary-verify: skipped — no canary fleet configured"
|
||||
exit 0
|
||||
fi
|
||||
bash scripts/canary-smoke.sh
|
||||
echo "ran=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Summary on failure
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
{
|
||||
echo "## Canary smoke FAILED"
|
||||
echo
|
||||
echo "Canary tenants rejected image \`staging-${{ steps.compute.outputs.sha }}\`."
|
||||
echo ":latest stays pinned to the prior good digest — prod is untouched."
|
||||
echo
|
||||
echo "Fix forward and merge again, or investigate the specific failed"
|
||||
echo "assertions in the canary-smoke step log above."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
promote-to-latest:
|
||||
# On green, calls the CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha> to promote the verified ECR image. This is the same
|
||||
# mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
|
||||
#
|
||||
# Pre-fix history: the old GHCR promote step used `crane tag` against
|
||||
# ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
|
||||
# image.yml had already migrated to ECR on 2026-05-07 (commit
|
||||
# 10e510f5). The GHCR tags were never updated, so this step was
|
||||
# silently promoting a stale GHCR image while actual prod tenants
|
||||
# pulled from ECR. Canary smoke tests were GHCR-targeted and could
|
||||
# not catch a broken ECR build.
|
||||
needs: canary-smoke
|
||||
if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
SHA: ${{ needs.canary-smoke.outputs.sha }}
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
# CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
|
||||
# Stored at the repo level so all workflows pick it up automatically.
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
# canary_slug pin: deploy the verified :staging-<sha> to the canary
|
||||
# first (soak 120s), then fan out to the rest of the fleet.
|
||||
CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
|
||||
SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
|
||||
BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
|
||||
steps:
|
||||
- name: Check CP credentials
|
||||
run: |
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
|
||||
echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Promote verified ECR image to :latest
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_TAG="staging-${SHA}"
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--argjson soak "${SOAK_SECONDS:-120}" \
|
||||
--argjson batch "${BATCH_SIZE:-3}" \
|
||||
--argjson dry false \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
if [ -n "${CANARY_SLUG:-}" ]; then
|
||||
BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
|
||||
fi
|
||||
|
||||
echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag: $TARGET_TAG"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
CURL_EXIT=$?
|
||||
set -e
|
||||
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
if [ "$HTTP_CODE" -ge 400 ]; then
|
||||
echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## Canary verified — :latest promoted via CP redeploy-fleet"
|
||||
echo ""
|
||||
echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
|
||||
echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
|
||||
echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "- **Batch size:** ${BATCH_SIZE:-3}"
|
||||
echo ""
|
||||
echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
|
||||
echo "The fleet's 5-minute health-check loop will pick up the update automatically."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,39 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Why a gate, not just discipline: PR #2536 pruned the manifest, but the
|
||||
# cascade list wasn't updated for ~weeks before someone (PR #2556)
|
||||
# noticed during an unrelated audit. During that window, codex never
|
||||
# rebuilt on a runtime publish. A structural gate catches the drift
|
||||
# the same day either file changes.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .github/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh
|
||||
@@ -0,0 +1,58 @@
|
||||
name: Check migration collisions
|
||||
|
||||
# Hard gate (#2341): fails a PR that adds a migration prefix already
|
||||
# claimed by the base branch or another open PR. Caught manually 2026-04-30
|
||||
# during PR #2276 rebase: 044_runtime_image_pins collided with
|
||||
# 044_platform_inbound_secret from RFC #2312. This workflow makes that
|
||||
# check automatic.
|
||||
#
|
||||
# Trigger model: pull_request only — there's no value running this on
|
||||
# pushes to staging or main (those are post-merge; the gate must fire
|
||||
# pre-merge to be useful). Path filter scopes to PRs that actually touch
|
||||
# migrations.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'scripts/ops/check_migration_collisions.py'
|
||||
- '.github/workflows/check-migration-collisions.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# gh pr list/diff need read access to other PRs
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Migration version collision check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Need history to diff against base ref
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect collisions
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: origin/${{ github.event.pull_request.base.ref }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.sha }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
# gh CLI uses GH_TOKEN from env
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Ensure the named base ref exists locally. checkout@v4 with
|
||||
# fetch-depth=0 pulls full history, but the explicit fetch is
|
||||
# cheap insurance against form-of-ref differences across runs.
|
||||
#
|
||||
# IMPORTANT: do NOT pass --depth=1 here. The script below uses
|
||||
# `git diff origin/<base>...<head>` (three-dot, merge-base form),
|
||||
# which fails with "fatal: no merge base" if the base ref is
|
||||
# shallow. The auto-promote staging→main PR (#2361) was blocked
|
||||
# by exactly this for ~5h on 2026-04-30 — the depth=1 fetch
|
||||
# overwrote checkout@v4's full-history clone with a shallow tip.
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
python3 scripts/ops/check_migration_collisions.py
|
||||
@@ -0,0 +1,443 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
# GitHub merge queue fires `merge_group` for the queue's pre-merge CI run.
|
||||
# Required so the queue gets a real check result instead of a false-green
|
||||
# from the absence of a triggered workflow. Safe to add unconditionally —
|
||||
# the event simply doesn't fire until the queue is enabled on the branch.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
# Cancel in-progress CI runs when a new commit arrives on the same ref.
|
||||
# This prevents stale runs from queuing behind each other. The merge_group
|
||||
# refs (refs/heads/gh-readonly-queue/...) get their own concurrency group
|
||||
# automatically because github.ref differs from the PR ref.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Detect which paths changed so downstream jobs can skip when only
|
||||
# docs/markdown files were modified.
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
platform: ${{ steps.check.outputs.platform }}
|
||||
canvas: ${{ steps.check.outputs.canvas }}
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
scripts: ${{ steps.check.outputs.scripts }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
run: |
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set by GitHub for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".github/workflows/ci.yml")
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) is a required check on staging. Always-run + per-step
|
||||
# gating (see Canvas (Next.js) for the rationale and the failure mode
|
||||
# this avoids).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: github.com/molecule-ai/molecule-cli
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
# offenders float to the top. Does NOT fail the build; the hard
|
||||
# gate is the threshold check below. (#1823)
|
||||
run: |
|
||||
echo "=== Per-file coverage (worst first) ==="
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
# 2. Per-file floor — non-test .go files in security-critical
|
||||
# paths with coverage <10% fail the build, UNLESS the file
|
||||
# path is listed in .coverage-allowlist.txt (acknowledged
|
||||
# historical debt with a tracking issue + expiry).
|
||||
run: |
|
||||
set -e
|
||||
TOTAL_FLOOR=25
|
||||
# Security-critical paths where a 0%-coverage file is a real risk.
|
||||
CRITICAL_PATHS=(
|
||||
"internal/handlers/tokens"
|
||||
"internal/handlers/workspace_provision"
|
||||
"internal/handlers/a2a_proxy"
|
||||
"internal/handlers/registry"
|
||||
"internal/handlers/secrets"
|
||||
"internal/middleware/wsauth"
|
||||
"internal/crypto"
|
||||
)
|
||||
|
||||
TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${TOTAL}%"
|
||||
if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
|
||||
echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
|
||||
> /tmp/perfile.txt
|
||||
|
||||
# Build allowlist — paths relative to workspace-server, one per line.
|
||||
# Lines starting with # are comments.
|
||||
ALLOWLIST=""
|
||||
if [ -f ../.coverage-allowlist.txt ]; then
|
||||
ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
WARNED=0
|
||||
for path in "${CRITICAL_PATHS[@]}"; do
|
||||
while read -r file pct; do
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !($pct < 10)}" || continue
|
||||
|
||||
# Strip the package-import prefix so we can match .coverage-allowlist.txt
|
||||
# entries written as paths relative to workspace-server/.
|
||||
# Handle both module paths: platform/workspace-server/... and platform/...
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
|
||||
WARNED=$((WARNED+1))
|
||||
else
|
||||
echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done < /tmp/perfile.txt
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED security-critical file(s) have <10% test coverage and are"
|
||||
echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
|
||||
echo "workspace provisioning — a 0% file here is the exact gap that let"
|
||||
echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
|
||||
echo " (a) add tests to raise coverage above 10%, or"
|
||||
echo " (b) add the path to .coverage-allowlist.txt with an expiry date"
|
||||
echo " and a tracking issue reference."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Canvas (Next.js) — required check, always runs. See platform-build
|
||||
# comment above for the rationale.
|
||||
#
|
||||
# Supersedes the canvas-build-noop pattern attempted in PR #2321: two
|
||||
# jobs sharing `name:` doesn't actually satisfy branch protection
|
||||
# because the SKIPPED check run sibling is treated as not-passed
|
||||
# regardless of how many SUCCESS siblings it has. Verified empirically
|
||||
# on PR #2314 — mergeStateStatus stayed BLOCKED until I collapsed to
|
||||
# a single-job-with-conditional-steps shape.
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
# #1815 — wires coverage into CI so we get a baseline visible on
|
||||
# every PR. No threshold gate yet; thresholds dial in (Step 3, also
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
# Per the inline comment in vitest.config.ts: "first land
|
||||
# observability so we can see the baseline, then dial in
|
||||
# thresholds + a hard gate" — this PR ships the observability half.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
# v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
|
||||
# currently supported on GHES`. Drop this pin when Gitea ships
|
||||
# the v4 protocol (tracked: post-Gitea-1.23 followup).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: canvas-coverage-${{ github.run_id }}
|
||||
path: canvas/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# MCP Server + SDK removed from CI — now in standalone repos:
|
||||
# - github.com/molecule-ai/molecule-mcp-server (npm CI)
|
||||
# - github.com/molecule-ai/molecule-sdk-python (PyPI CI)
|
||||
|
||||
# e2e-api job moved to .github/workflows/e2e-api.yml (issue #458).
|
||||
# It now has workflow-level concurrency (cancel-in-progress: false) so
|
||||
# new pushes queue the E2E run rather than cancelling it at the run level.
|
||||
|
||||
# Shellcheck (E2E scripts) — required check, always runs. See
|
||||
# platform-build for the rationale.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
# README quickstart — a shellcheck regression there silently breaks
|
||||
# new-user onboarding. scripts/ is intentionally excluded until its
|
||||
# pre-existing SC3040/SC3043 warnings are cleaned up.
|
||||
run: |
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
# Asserts every shell E2E test that calls `mktemp` also installs
|
||||
# an EXIT trap. Catches the /tmp-leak class — a missing trap
|
||||
# silently leaks scratch into CI runners (~10-100KB per run).
|
||||
# See tests/e2e/lint_cleanup_traps.sh for the rule + fix pattern.
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
# Pure-bash unit tests for E2E helper libs (lib/*.sh). These pin
|
||||
# behavior of dispatch logic that — when broken — silently masks as
|
||||
# "Could not resolve authentication method" only after a successful
|
||||
# tenant + workspace provision (PR #2571 incident, 2026-05-03). Add
|
||||
# new self-contained unit tests here as the lib/ directory grows;
|
||||
# tests requiring live CP/tenant credentials belong in the dedicated
|
||||
# e2e-staging-* workflows, not this job.
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: ubuntu-latest
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
env:
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
# Write body to a temp file — avoids backtick escaping in shell.
|
||||
cat > /tmp/deploy-reminder.md << 'BODY'
|
||||
## Canvas build passed ✅ — deploy required
|
||||
|
||||
The `publish-canvas-image` workflow is now building a fresh Docker image
|
||||
(`ghcr.io/molecule-ai/canvas:latest`) in the background.
|
||||
|
||||
Once it completes (~3–5 min), apply on the host machine with:
|
||||
```bash
|
||||
cd <runner-workspace>
|
||||
git pull origin main
|
||||
docker compose pull canvas && docker compose up -d canvas
|
||||
```
|
||||
|
||||
If you need to rebuild from local source instead (e.g. testing unreleased
|
||||
changes or a new `NEXT_PUBLIC_*` URL), use:
|
||||
```bash
|
||||
docker compose build canvas && docker compose up -d canvas
|
||||
```
|
||||
BODY
|
||||
printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
|
||||
"$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
|
||||
|
||||
# Gitea has no commit-comments API (no equivalent of
|
||||
# POST /repos/{owner}/{repo}/commits/{commit_sha}/comments).
|
||||
# Write to GITHUB_STEP_SUMMARY instead — both GitHub Actions and
|
||||
# Gitea Actions render this as the workflow run's summary page,
|
||||
# which is where operators look for post-deploy action items.
|
||||
# (#75 / PR-D)
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs. See platform-build
|
||||
# for the rationale.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. Rationale (issue #2790, after
|
||||
# the PR #2766 → PR #2771 cycle): the total floor averages ~6000
|
||||
# lines, so a single MCP file could regress to ~50% with no
|
||||
# complaint as long as other modules compensate. These five
|
||||
# files handle multi-tenant routing + auth + inbox dispatch —
|
||||
# a coverage drop here is the same risk shape as a Go-side
|
||||
# workspace-server token/secrets file dropping below 10%.
|
||||
#
|
||||
# Floor 75% sits below current actuals (80-96%) so this gate is
|
||||
# strictly additive — no existing PR fails. Ratchet plan in
|
||||
# COVERAGE_FLOOR.md.
|
||||
run: |
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text. --include uses fnmatch, and
|
||||
# the leading "*" allows the file to live anywhere under the
|
||||
# workspace root (today they sit at workspace/<name>.py).
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
# Match by top-level path key (e.g. "a2a_tools.py", not
|
||||
# "builtin_tools/a2a_tools.py" — different file at 100%).
|
||||
# The keys in coverage.json are paths relative to the run
|
||||
# cwd (workspace/), so the critical-path entry sits at the
|
||||
# bare basename.
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# SDK + plugin validation moved to standalone repo:
|
||||
# github.com/molecule-ai/molecule-sdk-python
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
name: Continuous synthetic E2E (staging)
|
||||
|
||||
# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
|
||||
# regressions visible only at runtime — schema drift, deployment-pipeline
|
||||
# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
|
||||
#
|
||||
# Why this gate exists:
|
||||
# PR-time CI catches code-level regressions but not deployment-time or
|
||||
# integration-time ones. Today's empirical data:
|
||||
# • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
|
||||
# JSON-RPC parse layer between sender and receiver. Visible only
|
||||
# to a sender exercising the full path.
|
||||
# • RFC #2312 chat upload — landed on staging-branch but never
|
||||
# reached staging tenants because publish-workspace-server-image
|
||||
# was main-only. Caught by manual dogfooding hours after deploy.
|
||||
# Both would have surfaced within 15-20 min of regression if a
|
||||
# continuous synth-E2E was running.
|
||||
#
|
||||
# Cadence: every 20 min (3x/hour). The script is conservatively
|
||||
# bounded at 10 min wall-clock; even on degraded staging it should
|
||||
# finish before the next firing. cron-overlap is guarded by the
|
||||
# concurrency group below.
|
||||
#
|
||||
# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
|
||||
# Plus a fresh tenant provisioned + torn down each run (Railway +
|
||||
# AWS pennies). Negligible.
|
||||
#
|
||||
# Failure handling: when the run fails, the workflow exits non-zero
|
||||
# and GitHub's standard email/notification path fires. Operators
|
||||
# can subscribe to this workflow's failure channel for paging-grade
|
||||
# alerting.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
|
||||
# Prior history: cron was '0,20,40' (2026-05-02) — only :00
|
||||
# ever survived. Bumped to '10,30,50' (2026-05-03) on the
|
||||
# theory that further-from-:00 wins. Empirically 2026-05-04
|
||||
# that ALSO dropped to ~60 min effective cadence (only ~1
|
||||
# schedule fire per hour — see molecule-core#2726). Detection
|
||||
# latency was claimed 20 min, actual 60 min.
|
||||
# 2. Avoid colliding with the existing :15 sweep-cf-orphans
|
||||
# and :45 sweep-cf-tunnels — both hit the CF API and we
|
||||
# don't want to fight for rate-limit tokens.
|
||||
# 3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
|
||||
# secrets, sweep-stale-e2e-orgs every :15) — multiple
|
||||
# overlapping cron registrations on the same minute is part
|
||||
# of what GH drops under load.
|
||||
# Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
|
||||
# lanes (1-3 min away from any other cron). Even with empirically-
|
||||
# observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
runtime:
|
||||
description: "Runtime to provision (claude-code = default + cheapest via MiniMax; langgraph = OpenAI-only; hermes = SDK-native path, slower)"
|
||||
required: false
|
||||
default: "claude-code"
|
||||
type: string
|
||||
model_slug:
|
||||
description: "Model id to provision the workspace with (default MiniMax-M2.7-highspeed; e.g. 'sonnet' to test direct Anthropic, 'openai/gpt-4o' for hermes)"
|
||||
required: false
|
||||
default: "MiniMax-M2.7-highspeed"
|
||||
type: string
|
||||
keep_org:
|
||||
description: "Skip teardown for post-mortem debugging (only manual dispatch — never set this for cron runs)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
# history. If you want auto-issue-on-fail, add a follow-up step that
|
||||
# uses gh issue create gated on `if: failure()`. Keeping the surface
|
||||
# minimal until that's actually wanted.
|
||||
|
||||
# Serialize so two firings can never overlap. Cron firing every 20 min
|
||||
# but scripts conservatively bounded at 10 min — overlap shouldn't
|
||||
# happen in steady state, but if a run hangs we don't want N more
|
||||
# stacking up.
|
||||
concurrency:
|
||||
group: continuous-synth-e2e
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
synth:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
# ssm-agent) runs from raw Ubuntu on every boot — none of it is
|
||||
# pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
|
||||
# across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
|
||||
# latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
|
||||
# A 12min budget leaves only ~2min for the workspace (which needs
|
||||
# ~3.5min for claude-code cold boot) on slow-apt days, blowing the
|
||||
# budget. 20min absorbs the worst tenant tail so the workspace probe
|
||||
# gets the full ~7min it needs even on a slow apt day. Real fix:
|
||||
# pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
# exhaustion class that took the canary down 2026-05-03 (#265).
|
||||
# Operators can pick langgraph / hermes via workflow_dispatch
|
||||
# when they specifically need to exercise the OpenAI or SDK-
|
||||
# native paths.
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
E2E_PROVISION_TIMEOUT_SECS: '600'
|
||||
# Slug suffix — namespaced "synth-" so these runs are
|
||||
# distinguishable from PR-driven runs in CP admin.
|
||||
E2E_RUN_ID: synth-${{ github.run_id }}
|
||||
# Forced false for cron; respected for manual dispatch
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
# tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
|
||||
# which key is present — MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so operators can dispatch with
|
||||
# E2E_RUNTIME=langgraph or =hermes and still have a working
|
||||
# canary path. The script picks the right blob shape based on
|
||||
# which key is non-empty.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
run: |
|
||||
# Hard-fail on missing secret REGARDLESS of trigger. Previously
|
||||
# this step soft-skipped on workflow_dispatch via `exit 0`, but
|
||||
# `exit 0` only ends the STEP — subsequent steps still ran with
|
||||
# the empty secret, the synth script fell through to the wrong
|
||||
# SECRETS_JSON branch, and the canary failed 5 min later with a
|
||||
# confusing "Agent error (Exception)" instead of the clean
|
||||
# "secret missing" message at the top. Caught 2026-05-04 by
|
||||
# dispatched run 25296530706: claude-code + missing MINIMAX
|
||||
# silently used OpenAI keys but kept model=MiniMax-M2.7, then
|
||||
# the workspace 401'd against MiniMax once it tried to call.
|
||||
# Fix: exit 1 in both cron and dispatch paths. Operators who
|
||||
# want to verify a YAML change without setting up the secret
|
||||
# can read the verify-secrets step's stderr — the failure is
|
||||
# itself the verification signal.
|
||||
if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
# langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_KEY).
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install required tools
|
||||
run: |
|
||||
# The script depends on jq + curl (already on ubuntu-latest)
|
||||
# and python3 (likewise). Verify they're all present so we
|
||||
# fail fast on a runner image regression rather than mid-script.
|
||||
for cmd in jq curl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "::error::required tool '$cmd' not on PATH — runner image regression?"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
- name: Run synthetic E2E
|
||||
# The script handles its own teardown via EXIT trap; even on
|
||||
# failure (timeout, assertion), the org is deprovisioned and
|
||||
# leaks are reported. Exit code propagates from the script.
|
||||
run: |
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
- name: Failure summary
|
||||
# Runs only on failure. Adds a job summary so the workflow run
|
||||
# page shows a quick "what happened" instead of forcing readers
|
||||
# to scroll through script output.
|
||||
if: failure()
|
||||
run: |
|
||||
{
|
||||
echo "## Continuous synth E2E failed"
|
||||
echo ""
|
||||
echo "**Run ID:** ${{ github.run_id }}"
|
||||
echo "**Trigger:** ${{ github.event_name }}"
|
||||
echo "**Runtime:** ${E2E_RUNTIME}"
|
||||
echo "**Slug:** synth-${{ github.run_id }}"
|
||||
echo ""
|
||||
echo "### What this means"
|
||||
echo ""
|
||||
echo "Staging just regressed on a path that previously worked. Likely classes:"
|
||||
echo "- Schema mismatch between sender and receiver (#2345 class)"
|
||||
echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
|
||||
echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
|
||||
echo "- Staging-CP env var rotation"
|
||||
echo ""
|
||||
echo "### Next steps"
|
||||
echo ""
|
||||
echo "1. Check the script output above for the assertion that failed"
|
||||
echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
|
||||
echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
|
||||
echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user