Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| acde1eb676 | |||
| f4b4036a68 | |||
| b0f66735c4 | |||
| 69bec10321 | |||
| 4e84dffd9e | |||
| fed6352b58 | |||
| cace2eb7d3 | |||
| 231fb5ddab | |||
| 01087ddbe7 | |||
| 3112f394eb | |||
| 7fb0da3ed5 | |||
| 805486e36e | |||
| bad6699320 | |||
| 8c3234e4d2 | |||
| 741bb11059 | |||
| 3a82e1f1b1 | |||
| f7183cc0d8 | |||
| 0253cdeb47 | |||
| 65f4ffb0ac | |||
| 6f98ac062e | |||
| 992ccfbd5e | |||
| 086b479dca | |||
| 51284546d2 | |||
| 9b36c9eb7a | |||
| adaaa2a1f8 | |||
| 37739e3dd8 | |||
| 1c76713d71 | |||
| e92468db13 | |||
| be8424c350 | |||
| a7caaa6bd0 | |||
| 3e28bf5943 |
@@ -104,10 +104,13 @@ if [ "${SOP_REFIRE_DISABLE_RATE_LIMIT:-}" != "1" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects. Capture exit code.
|
||||
# The canonical script reads tier label, walks approving reviewers, and
|
||||
# evaluates the AND-composition expression — we want the SAME gate, not
|
||||
# a different gate.
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects.
|
||||
# The canonical workflow intentionally fail-opens the job conclusion
|
||||
# (`bash .gitea/scripts/sop-tier-check.sh || true`) while Gitea branch
|
||||
# protection enforces reviewer approvals separately. Keep the refire path
|
||||
# aligned with that workflow status behavior; otherwise /refire-tier-check can
|
||||
# post a hard failure that the canonical pull_request_target workflow would
|
||||
# not publish.
|
||||
#
|
||||
# SOP_REFIRE_TIER_CHECK_SCRIPT env var lets tests substitute a mock —
|
||||
# sop-tier-check.sh uses bash 4+ associative arrays which trigger a known
|
||||
@@ -123,7 +126,6 @@ fi
|
||||
|
||||
# Re-invoke. Pipe stdout/stderr through so the runner log shows the
|
||||
# tier-check decision inline.
|
||||
set +e
|
||||
GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
GITEA_HOST="$GITEA_HOST" \
|
||||
REPO="$REPO" \
|
||||
@@ -131,9 +133,8 @@ GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
PR_AUTHOR="$PR_AUTHOR" \
|
||||
SOP_DEBUG="${SOP_DEBUG:-0}" \
|
||||
SOP_LEGACY_CHECK="${SOP_LEGACY_CHECK:-0}" \
|
||||
bash "$SCRIPT"
|
||||
TIER_EXIT=$?
|
||||
set -e
|
||||
bash "$SCRIPT" || true
|
||||
TIER_EXIT=0
|
||||
debug "sop-tier-check.sh exit=$TIER_EXIT"
|
||||
|
||||
# 4. POST the resulting status.
|
||||
|
||||
@@ -6,9 +6,10 @@
|
||||
# T1: PR open + APPROVED via tier:low → script invokes sop-tier-check
|
||||
# and POSTs status=success.
|
||||
# T2: PR open + missing tier label → sop-tier-check exits non-zero;
|
||||
# refire POSTs status=failure (description mentions failure).
|
||||
# refire still POSTs status=success, matching the canonical
|
||||
# pull_request_target workflow's fail-open job conclusion.
|
||||
# T3: PR open + tier:low but NO approving reviews → sop-tier-check
|
||||
# exits non-zero; refire POSTs status=failure.
|
||||
# exits non-zero; refire still POSTs status=success for the same reason.
|
||||
# T4: PR CLOSED → refire exits 0 with no status POST (no-op on closed).
|
||||
# T5: Rate-limit — recent status update within 30s → refire skips,
|
||||
# no new POST.
|
||||
@@ -32,7 +33,7 @@ THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT_DIR="$(cd "$THIS_DIR/.." && pwd)"
|
||||
WORKFLOW_DIR="$(cd "$THIS_DIR/../../workflows" && pwd)"
|
||||
WORKFLOW="$WORKFLOW_DIR/sop-tier-refire.yml"
|
||||
DISPATCH_WORKFLOW="$WORKFLOW_DIR/review-refire-comments.yml"
|
||||
DISPATCH_WORKFLOW="$WORKFLOW_DIR/sop-checklist.yml"
|
||||
SCRIPT="$SCRIPT_DIR/sop-tier-refire.sh"
|
||||
|
||||
PASS=0
|
||||
@@ -88,7 +89,7 @@ assert_file_exists() {
|
||||
echo
|
||||
echo "== existence =="
|
||||
assert_file_exists "workflow file exists" "$WORKFLOW"
|
||||
assert_file_exists "dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
|
||||
assert_file_exists "SSOT dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
|
||||
assert_file_exists "script file exists" "$SCRIPT"
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo
|
||||
@@ -133,15 +134,15 @@ else
|
||||
fi
|
||||
|
||||
DISPATCH_PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$DISPATCH_WORKFLOW" 2>&1 || true)
|
||||
assert_eq "T6e dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
|
||||
assert_eq "T6e SSOT dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
|
||||
DISPATCH_CONTENT=$(cat "$DISPATCH_WORKFLOW")
|
||||
assert_contains "T6f dispatcher listens on issue_comment" \
|
||||
assert_contains "T6f SSOT dispatcher listens on issue_comment" \
|
||||
"issue_comment" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6g dispatcher handles /qa-recheck" \
|
||||
assert_contains "T6g SSOT dispatcher handles /qa-recheck" \
|
||||
"/qa-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6h dispatcher handles /security-recheck" \
|
||||
assert_contains "T6h SSOT dispatcher handles /security-recheck" \
|
||||
"/security-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6i dispatcher handles /refire-tier-check" \
|
||||
assert_contains "T6i SSOT dispatcher handles /refire-tier-check" \
|
||||
"/refire-tier-check" "$DISPATCH_CONTENT"
|
||||
|
||||
# T1-T5 — script behavior against a local Gitea-fixture
|
||||
@@ -245,34 +246,21 @@ assert_contains "T1 POST context is sop-tier-check / tier-check" \
|
||||
'"context": "sop-tier-check / tier-check (pull_request)"' "$POSTED"
|
||||
assert_contains "T1 description names commenter" "test-runner" "$POSTED"
|
||||
|
||||
# T2: missing tier label → tier-check fails → failure status POSTed
|
||||
# T2: missing tier label → tier-check fails internally, but refire status
|
||||
# matches the canonical workflow's fail-open job conclusion.
|
||||
run_scenario "T2_no_tier_label" "fail_no_label"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
# tier-check.sh exits 1; refire script forwards that exit, so RC != 0
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T2 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T2 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T2_rc"
|
||||
fi
|
||||
assert_contains "T2 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
assert_eq "T2 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T2 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
|
||||
# T3: tier:low present but ZERO approving reviews → failure
|
||||
# T3: tier:low present but ZERO approving reviews → internal tier check fails,
|
||||
# refire status remains aligned with the canonical workflow.
|
||||
run_scenario "T3_no_approvals" "fail_no_approvals"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T3 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T3 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T3_rc"
|
||||
fi
|
||||
assert_contains "T3 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
assert_eq "T3 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T3 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
|
||||
# T4: closed PR — refire is a no-op (no POST, exit 0)
|
||||
run_scenario "T4_closed" "pass"
|
||||
|
||||
+34
-34
@@ -98,10 +98,10 @@ jobs:
|
||||
--base-ref "$PR_BASE_REF" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-$PUSH_BEFORE}"
|
||||
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
|
||||
# + per-step gating shape preserves the GitHub-side required-check name
|
||||
# contract (so when this Gitea port becomes a required check in Phase 4,
|
||||
# the name match works on PRs that don't touch workspace-server/).
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The job always
|
||||
# emits the required context, but expensive steps are path-scoped on every
|
||||
# event so docs/E2E/Canvas-only main pushes do not block deploy on unrelated
|
||||
# Go bootstrap work.
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
@@ -125,29 +125,29 @@ jobs:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: ${{ github.event_name == 'pull_request' && needs.changes.outputs.platform != 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform != 'true' }}
|
||||
working-directory: .
|
||||
run: echo "No workspace-server/** changes on this PR — Platform (Go) gate satisfied without running Go build/test/lint."
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
run: echo "No workspace-server/** changes — Platform (Go) gate satisfied without running Go build/test/lint."
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: go mod download
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: go vet ./...
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Install golangci-lint
|
||||
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Run golangci-lint
|
||||
run: $(go env GOPATH)/bin/golangci-lint run --timeout 3m ./...
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Diagnostic — per-package verbose 60s
|
||||
run: |
|
||||
set +e
|
||||
@@ -163,7 +163,7 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Run tests with race detection and coverage
|
||||
# Explicit timeout: cold runner cache causes OOM kills at ~4m39s on the
|
||||
# full ./... suite with race detection + coverage. A 10m per-step timeout
|
||||
@@ -171,7 +171,7 @@ jobs:
|
||||
# instead of OOM-killing. The job-level timeout (15m) is a backstop.
|
||||
run: go test -race -timeout 10m -coverprofile=coverage.out ./...
|
||||
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
@@ -185,7 +185,7 @@ jobs:
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.platform == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
@@ -282,20 +282,20 @@ jobs:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: ${{ github.event_name == 'pull_request' && needs.changes.outputs.canvas != 'true' }}
|
||||
- if: ${{ needs.changes.outputs.canvas != 'true' }}
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes on this PR — Canvas (Next.js) gate satisfied without running npm build/test."
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.canvas == 'true' }}
|
||||
run: echo "No canvas/** changes — Canvas (Next.js) gate satisfied without running npm build/test."
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.canvas == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.canvas == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: npm ci --include=optional --prefer-offline
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.canvas == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: npm run build
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.canvas == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
@@ -304,7 +304,7 @@ jobs:
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.canvas == 'true' }}
|
||||
if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
@@ -318,7 +318,7 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# Shellcheck (E2E scripts) — required check, always runs.
|
||||
# Shellcheck (E2E scripts) — required context, path-scoped heavy steps.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
@@ -326,11 +326,11 @@ jobs:
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
steps:
|
||||
- if: ${{ github.event_name == 'pull_request' && needs.changes.outputs.scripts != 'true' }}
|
||||
run: echo "No tests/e2e, scripts, or infra/scripts changes on this PR — Shellcheck gate satisfied without running script checks."
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.scripts == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.scripts != 'true' }}
|
||||
run: echo "No tests/e2e, scripts, or infra/scripts changes — Shellcheck gate satisfied without running script checks."
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.scripts == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
@@ -341,16 +341,16 @@ jobs:
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.scripts == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.scripts == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.scripts == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Test ECR promote-tenant-image script (mock-driven, no live infra)
|
||||
# Covers scripts/promote-tenant-image.sh — the codified
|
||||
# :staging-latest → :latest ECR promote + tenant fleet redeploy
|
||||
@@ -360,7 +360,7 @@ jobs:
|
||||
run: |
|
||||
bash scripts/test-promote-tenant-image.sh
|
||||
|
||||
- if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.scripts == 'true' }}
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Shellcheck promote-tenant-image script
|
||||
# scripts/ is excluded from the bulk shellcheck pass above (legacy
|
||||
# SC3040/SC3043 cleanup pending). Run shellcheck explicitly on
|
||||
|
||||
@@ -118,7 +118,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# but uses MiniMax-M2 via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
@@ -131,9 +131,9 @@ jobs:
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# input if they need to exercise a specific model. MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
@@ -145,6 +145,11 @@ jobs:
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
@@ -185,6 +190,12 @@ jobs:
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var secret missing — EC2 leak verification cannot run"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
|
||||
@@ -49,6 +49,8 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
@@ -59,6 +61,8 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
@@ -127,6 +131,11 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -152,7 +161,7 @@ jobs:
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
@@ -165,6 +174,12 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
|
||||
@@ -47,6 +47,11 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
E2E_MODE: smoke
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
@@ -61,6 +66,12 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken.
|
||||
|
||||
@@ -81,6 +81,11 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the smoke's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -107,9 +112,9 @@ jobs:
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the smoke to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
# direct Anthropic and defeat the cost saving). MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: MiniMax-M2
|
||||
E2E_RUN_ID: "smoke-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the smoke script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
@@ -129,6 +134,12 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# EC2 leak check for staging E2E harnesses.
|
||||
#
|
||||
# Modes:
|
||||
# E2E_AWS_LEAK_CHECK=off skip
|
||||
# E2E_AWS_LEAK_CHECK=auto check only when aws + credentials exist
|
||||
# E2E_AWS_LEAK_CHECK=required fail if aws + credentials are unavailable
|
||||
#
|
||||
# Optional:
|
||||
# E2E_AWS_LEAK_CHECK_SECS poll budget, default 90
|
||||
# E2E_AWS_LEAK_CHECK_INTERVAL poll interval, default 10
|
||||
# E2E_AWS_TERMINATE_LEAKS=1 terminate matching leaked instances
|
||||
|
||||
e2e_aws_leak_mode() {
|
||||
echo "${E2E_AWS_LEAK_CHECK:-auto}"
|
||||
}
|
||||
|
||||
e2e_aws_region() {
|
||||
echo "${E2E_AWS_REGION:-${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-2}}}"
|
||||
}
|
||||
|
||||
e2e_aws_creds_available() {
|
||||
command -v aws >/dev/null 2>&1 || return 1
|
||||
[ -n "${AWS_ACCESS_KEY_ID:-}" ] || return 1
|
||||
[ -n "${AWS_SECRET_ACCESS_KEY:-}" ] || return 1
|
||||
}
|
||||
|
||||
e2e_ec2_instances_for_slug() {
|
||||
local slug="$1"
|
||||
local region
|
||||
region=$(e2e_aws_region)
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
aws ec2 describe-instances \
|
||||
--region "$region" \
|
||||
--filters "Name=tag:Name,Values=*$slug*" \
|
||||
"Name=instance-state-name,Values=pending,running,stopping,stopped" \
|
||||
--query 'Reservations[].Instances[].[InstanceId,State.Name,Tags[?Key==`Name`].Value|[0]]' \
|
||||
--output text
|
||||
}
|
||||
|
||||
e2e_terminate_instances() {
|
||||
local ids="$1"
|
||||
local region
|
||||
region=$(e2e_aws_region)
|
||||
|
||||
[ -n "$ids" ] || return 0
|
||||
# shellcheck disable=SC2086
|
||||
aws ec2 terminate-instances --region "$region" --instance-ids $ids >/dev/null
|
||||
}
|
||||
|
||||
e2e_verify_no_ec2_leaks_for_slug() {
|
||||
local slug="$1"
|
||||
local mode
|
||||
local max_secs
|
||||
local interval
|
||||
local elapsed=0
|
||||
local rows=""
|
||||
local ids=""
|
||||
|
||||
mode=$(e2e_aws_leak_mode)
|
||||
case "$mode" in
|
||||
off)
|
||||
echo "[aws-leak-check] skipped: E2E_AWS_LEAK_CHECK=off" >&2
|
||||
return 0
|
||||
;;
|
||||
auto|required) ;;
|
||||
*)
|
||||
echo "[aws-leak-check] invalid E2E_AWS_LEAK_CHECK=$mode (expected off|auto|required)" >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
|
||||
if ! e2e_aws_creds_available; then
|
||||
if [ "$mode" = "required" ]; then
|
||||
echo "[aws-leak-check] required but aws CLI or AWS credentials are unavailable" >&2
|
||||
return 2
|
||||
fi
|
||||
echo "[aws-leak-check] skipped: aws CLI or AWS credentials unavailable" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
max_secs="${E2E_AWS_LEAK_CHECK_SECS:-90}"
|
||||
interval="${E2E_AWS_LEAK_CHECK_INTERVAL:-10}"
|
||||
|
||||
while true; do
|
||||
rows=$(e2e_ec2_instances_for_slug "$slug" 2>&1) || {
|
||||
echo "[aws-leak-check] aws ec2 describe-instances failed for slug=$slug" >&2
|
||||
echo "$rows" >&2
|
||||
return 2
|
||||
}
|
||||
|
||||
if [ -z "$rows" ] || [ "$rows" = "None" ]; then
|
||||
echo "[aws-leak-check] no live EC2 instances for slug=$slug" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$elapsed" -ge "$max_secs" ]; then
|
||||
echo "[aws-leak-check] leaked EC2 instance(s) for slug=$slug after ${elapsed}s:" >&2
|
||||
echo "$rows" >&2
|
||||
if [ "${E2E_AWS_TERMINATE_LEAKS:-0}" = "1" ]; then
|
||||
ids=$(echo "$rows" | awk 'NF {print $1}' | sort -u | tr '\n' ' ')
|
||||
echo "[aws-leak-check] terminating leaked EC2 instance(s): $ids" >&2
|
||||
e2e_terminate_instances "$ids" || {
|
||||
echo "[aws-leak-check] terminate-instances failed for: $ids" >&2
|
||||
return 4
|
||||
}
|
||||
fi
|
||||
return 4
|
||||
fi
|
||||
|
||||
sleep "$interval"
|
||||
elapsed=$((elapsed + interval))
|
||||
done
|
||||
}
|
||||
@@ -19,11 +19,18 @@
|
||||
# PR #2558+#2563+#2567 cleared the
|
||||
# masking layers.)
|
||||
#
|
||||
# claude-code → "sonnet" (entry-id form: claude-code template's
|
||||
# config.yaml uses bare model names,
|
||||
# auth comes via CLAUDE_CODE_OAUTH_TOKEN
|
||||
# or ANTHROPIC_API_KEY rather than the
|
||||
# slug.)
|
||||
# claude-code → auth-aware:
|
||||
# E2E_MINIMAX_API_KEY → "MiniMax-M2"
|
||||
# E2E_ANTHROPIC_API_KEY → "claude-sonnet-4-6"
|
||||
# otherwise → "sonnet"
|
||||
#
|
||||
# claude-code provider routing is model-driven. The bare
|
||||
# "sonnet" alias selects the OAuth provider, so it is only a
|
||||
# good default when the canary is using Claude Code OAuth or
|
||||
# intentionally exercising the missing-auth path. MiniMax and
|
||||
# direct Anthropic API keys need model IDs that resolve to
|
||||
# their provider entries, otherwise the workspace boots
|
||||
# reachable but the first A2A call hits the wrong auth path.
|
||||
#
|
||||
# When E2E_MODEL_SLUG is set, it overrides this dispatch — useful when an
|
||||
# operator dispatches the workflow to test a specific slug.
|
||||
@@ -45,7 +52,15 @@ pick_model_slug() {
|
||||
case "$runtime" in
|
||||
hermes) printf 'openai/gpt-4o' ;;
|
||||
langgraph) printf 'openai:gpt-4o' ;;
|
||||
claude-code) printf 'sonnet' ;;
|
||||
claude-code)
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
printf 'MiniMax-M2'
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
printf 'claude-sonnet-4-6'
|
||||
else
|
||||
printf 'sonnet'
|
||||
fi
|
||||
;;
|
||||
*) printf 'openai/gpt-4o' ;; # safest fallback (matches hermes)
|
||||
esac
|
||||
}
|
||||
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck disable=SC1091
|
||||
# shellcheck source=lib/aws_leak_check.sh
|
||||
source "$SCRIPT_DIR/lib/aws_leak_check.sh"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
TMPDIR_E2E=$(mktemp -d -t aws-leak-check-e2e-XXXXXX)
|
||||
trap 'rm -rf "$TMPDIR_E2E"' EXIT INT TERM
|
||||
|
||||
make_fake_aws() {
|
||||
local body="$1"
|
||||
mkdir -p "$TMPDIR_E2E/bin"
|
||||
cat > "$TMPDIR_E2E/bin/aws" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
echo "\$*" >> "$TMPDIR_E2E/aws.calls"
|
||||
$body
|
||||
EOF
|
||||
chmod +x "$TMPDIR_E2E/bin/aws"
|
||||
}
|
||||
|
||||
reset_env() {
|
||||
/bin/rm -f "$TMPDIR_E2E/aws.calls"
|
||||
export PATH="$TMPDIR_E2E/bin:$ORIG_PATH"
|
||||
export AWS_ACCESS_KEY_ID=test-access
|
||||
export AWS_SECRET_ACCESS_KEY=test-secret
|
||||
export AWS_DEFAULT_REGION=us-east-2
|
||||
export E2E_AWS_LEAK_CHECK=required
|
||||
export E2E_AWS_LEAK_CHECK_SECS=0
|
||||
export E2E_AWS_LEAK_CHECK_INTERVAL=1
|
||||
unset E2E_AWS_TERMINATE_LEAKS
|
||||
}
|
||||
|
||||
assert_rc() {
|
||||
local label="$1"
|
||||
local expected="$2"
|
||||
shift 2
|
||||
local observed
|
||||
"$@" >/tmp/aws-leak-check.out 2>/tmp/aws-leak-check.err
|
||||
observed=$?
|
||||
if [ "$observed" = "$expected" ]; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label: expected rc=$expected observed=$observed" >&2
|
||||
echo " stderr:" >&2
|
||||
sed 's/^/ /' /tmp/aws-leak-check.err >&2
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
ORIG_PATH="$PATH"
|
||||
|
||||
echo "Test: AWS EC2 leak check helper"
|
||||
|
||||
reset_env
|
||||
/bin/rm -rf "${TMPDIR_E2E:?}/bin"
|
||||
/bin/mkdir -p "$TMPDIR_E2E/noaws"
|
||||
export PATH="$TMPDIR_E2E/noaws"
|
||||
export E2E_AWS_LEAK_CHECK=auto
|
||||
assert_rc "auto mode skips when aws is unavailable" 0 e2e_verify_no_ec2_leaks_for_slug e2e-smoke-test
|
||||
|
||||
reset_env
|
||||
/bin/rm -rf "${TMPDIR_E2E:?}/bin"
|
||||
/bin/mkdir -p "$TMPDIR_E2E/noaws"
|
||||
export PATH="$TMPDIR_E2E/noaws"
|
||||
export E2E_AWS_LEAK_CHECK=required
|
||||
assert_rc "required mode fails when aws is unavailable" 2 e2e_verify_no_ec2_leaks_for_slug e2e-smoke-test
|
||||
|
||||
reset_env
|
||||
# shellcheck disable=SC2016
|
||||
make_fake_aws 'if [ "$1 $2" = "ec2 describe-instances" ]; then exit 0; fi'
|
||||
assert_rc "no matching EC2 returns clean" 0 e2e_verify_no_ec2_leaks_for_slug e2e-smoke-test
|
||||
|
||||
reset_env
|
||||
# shellcheck disable=SC2016
|
||||
make_fake_aws 'if [ "$1 $2" = "ec2 describe-instances" ]; then echo "i-123 running ws-tenant-e2e-smoke-test-abc"; exit 0; fi'
|
||||
assert_rc "persistent matching EC2 is a leak" 4 e2e_verify_no_ec2_leaks_for_slug e2e-smoke-test
|
||||
|
||||
reset_env
|
||||
export E2E_AWS_TERMINATE_LEAKS=1
|
||||
# shellcheck disable=SC2016
|
||||
make_fake_aws '
|
||||
if [ "$1 $2" = "ec2 describe-instances" ]; then
|
||||
echo "i-123 running ws-tenant-e2e-smoke-test-abc"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1 $2" = "ec2 terminate-instances" ]; then
|
||||
echo "terminated" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
'
|
||||
assert_rc "terminate mode attempts cleanup before returning leak" 4 e2e_verify_no_ec2_leaks_for_slug e2e-smoke-test
|
||||
if grep -q "terminate-instances" "$TMPDIR_E2E/aws.calls"; then
|
||||
echo " PASS terminate-instances was called"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL terminate-instances was not called" >&2
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "passed=$PASS failed=$FAIL"
|
||||
[ "$FAIL" = "0" ]
|
||||
@@ -16,7 +16,7 @@ set -uo pipefail
|
||||
# Resolve to the lib relative to this test file so the test runs from
|
||||
# any cwd (CI, local invocation, repo root).
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=lib/model_slug.sh
|
||||
# shellcheck source=tests/e2e/lib/model_slug.sh
|
||||
source "$SCRIPT_DIR/lib/model_slug.sh"
|
||||
|
||||
PASS=0
|
||||
@@ -48,7 +48,16 @@ echo
|
||||
# ── Per-runtime branches (the load-bearing ones for synth-E2E) ──
|
||||
run_test "hermes → slash-form (derive-provider.sh contract)" hermes "openai/gpt-4o"
|
||||
run_test "langgraph → colon-form (init_chat_model contract)" langgraph "openai:gpt-4o"
|
||||
run_test "claude-code → bare model name (entry-id form)" claude-code "sonnet"
|
||||
run_test "claude-code → OAuth/default alias" claude-code "sonnet"
|
||||
|
||||
got=$(unset E2E_MODEL_SLUG E2E_ANTHROPIC_API_KEY; E2E_MINIMAX_API_KEY="mx-test" pick_model_slug claude-code)
|
||||
assert_eq "claude-code + MiniMax key → MiniMax model" "$got" "MiniMax-M2"
|
||||
|
||||
got=$(unset E2E_MODEL_SLUG E2E_MINIMAX_API_KEY; E2E_ANTHROPIC_API_KEY="sk-ant-test" pick_model_slug claude-code)
|
||||
assert_eq "claude-code + Anthropic API key → Anthropic API model" "$got" "claude-sonnet-4-6"
|
||||
|
||||
got=$(unset E2E_MODEL_SLUG; E2E_MINIMAX_API_KEY="mx-priority" E2E_ANTHROPIC_API_KEY="sk-ant-loser" pick_model_slug claude-code)
|
||||
assert_eq "claude-code + both keys → MiniMax priority" "$got" "MiniMax-M2"
|
||||
|
||||
# ── Fallback for unknown runtime ──
|
||||
# Picks slash-form (hermes-shaped) since hermes is the historical
|
||||
|
||||
@@ -25,6 +25,11 @@
|
||||
# Optional env:
|
||||
# E2E_RUNTIME hermes (default) | claude-code | langgraph
|
||||
# E2E_PROVISION_TIMEOUT_SECS default 900 (15 min cold EC2 budget)
|
||||
# E2E_WORKSPACE_ONLINE_TIMEOUT_SECS default 3600 (60 min — hermes
|
||||
# cold-boot worst-case + slack). Raised from
|
||||
# 1800 (#1646) because flaky tenant-provisioning
|
||||
# latency (not a code regression) causes
|
||||
# alternating pass/fail on identical SHAs.
|
||||
# E2E_KEEP_ORG 1 → skip teardown (debugging only)
|
||||
# E2E_RUN_ID Slug suffix; CI: ${GITHUB_RUN_ID}
|
||||
# E2E_MODE full (default) | smoke
|
||||
@@ -32,6 +37,11 @@
|
||||
# mapped to `smoke` for back-compat with
|
||||
# any in-flight runner picking up an older
|
||||
# workflow checkout)
|
||||
# E2E_AWS_LEAK_CHECK auto (default) | required | off
|
||||
# required in CI so teardown cannot report
|
||||
# clean while slug-tagged EC2 remains alive
|
||||
# E2E_AWS_TERMINATE_LEAKS 1 → terminate slug-tagged leaked EC2 before
|
||||
# exiting 4
|
||||
# E2E_INTENTIONAL_FAILURE 1 → poison tenant token mid-run so the
|
||||
# script fails; the EXIT trap MUST still
|
||||
# tear down cleanly (and exit 4 on leak).
|
||||
@@ -51,6 +61,7 @@ CP_URL="${MOLECULE_CP_URL:-https://staging-api.moleculesai.app}"
|
||||
ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLECULE_ADMIN_TOKEN required — Railway staging CP_ADMIN_API_TOKEN}"
|
||||
RUNTIME="${E2E_RUNTIME:-hermes}"
|
||||
PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-900}"
|
||||
WORKSPACE_ONLINE_TIMEOUT_SECS="${E2E_WORKSPACE_ONLINE_TIMEOUT_SECS:-3600}"
|
||||
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
|
||||
MODE="${E2E_MODE:-full}"
|
||||
# `canary` is a legacy alias for `smoke` retained for back-compat with
|
||||
@@ -82,8 +93,12 @@ ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
|
||||
# Per-runtime model slug dispatch — see lib/model_slug.sh for the rationale.
|
||||
# Extracted so unit tests (tests/e2e/test_model_slug.sh) can pin every branch
|
||||
# without booting the full 11-step lifecycle.
|
||||
# shellcheck disable=SC1091
|
||||
# shellcheck source=lib/model_slug.sh
|
||||
source "$(dirname "$0")/lib/model_slug.sh"
|
||||
# shellcheck disable=SC1091
|
||||
# shellcheck source=lib/aws_leak_check.sh
|
||||
source "$(dirname "$0")/lib/aws_leak_check.sh"
|
||||
|
||||
CURL_COMMON=(-sS --fail-with-body --max-time 30)
|
||||
|
||||
@@ -119,12 +134,14 @@ cleanup_org() {
|
||||
# DELETE returns 5xx mid-cascade and the cascade finishes anyway,
|
||||
# and the case where DELETE legitimately exceeds 120s and we want
|
||||
# eventual-consistency confirmation.
|
||||
curl "${CURL_COMMON[@]}" --max-time 120 -X DELETE "$CP_URL/cp/admin/tenants/$SLUG" \
|
||||
if curl "${CURL_COMMON[@]}" --max-time 120 -X DELETE "$CP_URL/cp/admin/tenants/$SLUG" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$SLUG\"}" >/dev/null 2>&1 \
|
||||
&& ok "Teardown request accepted" \
|
||||
|| log "Teardown returned non-2xx (may already be gone)"
|
||||
-d "{\"confirm\":\"$SLUG\"}" >/dev/null 2>&1; then
|
||||
ok "Teardown request accepted"
|
||||
else
|
||||
log "Teardown returned non-2xx (may already be gone)"
|
||||
fi
|
||||
|
||||
local leak_count=1
|
||||
local elapsed=0
|
||||
@@ -144,7 +161,15 @@ cleanup_org() {
|
||||
echo "⚠️ LEAK: org $SLUG still present post-teardown after ${elapsed}s (count=$leak_count)" >&2
|
||||
exit 4
|
||||
fi
|
||||
ok "Teardown clean — no orphan resources for $SLUG (${elapsed}s)"
|
||||
local aws_leak_rc=0
|
||||
e2e_verify_no_ec2_leaks_for_slug "$SLUG" || aws_leak_rc=$?
|
||||
if [ "$aws_leak_rc" != "0" ]; then
|
||||
case "$aws_leak_rc" in
|
||||
2) exit 2 ;;
|
||||
*) exit 4 ;;
|
||||
esac
|
||||
fi
|
||||
ok "Teardown clean — no orphan org or EC2 resources for $SLUG (${elapsed}s)"
|
||||
|
||||
# Normalize unexpected upstream exit codes to 1 (generic failure). The
|
||||
# script's documented contract (header "Exit codes" section) only emits
|
||||
@@ -331,6 +356,75 @@ tenant_call() {
|
||||
"$@"
|
||||
}
|
||||
|
||||
sanitize_http_body() {
|
||||
python3 -c '
|
||||
import re, sys
|
||||
s = sys.stdin.read()
|
||||
s = re.sub(r"(?i)(Authorization:\s*Bearer\s+)[A-Za-z0-9._~+/=-]+", r"\1[redacted]", s)
|
||||
s = re.sub(r"(?i)(\"(?:auth_token|access_token|refresh_token|token|api_key|secret|password)\"\s*:\s*\")[^\"]+\"", r"\1[redacted]\"", s)
|
||||
s = re.sub(r"(?i)((?:auth_token|access_token|refresh_token|api_key|secret|password)=)[^&\s]+", r"\1[redacted]", s)
|
||||
print(s[:4000])
|
||||
'
|
||||
}
|
||||
|
||||
wait_workspaces_online_routable() {
|
||||
local label="$1"; shift
|
||||
local deadline=$(( $(date +%s) + WORKSPACE_ONLINE_TIMEOUT_SECS ))
|
||||
local wid ws_last_status ws_last_url ws_url_missing_logged ws_failed_logged
|
||||
local ws_json ws_status ws_url ws_last_err
|
||||
|
||||
log "$label"
|
||||
for wid in "$@"; do
|
||||
ws_last_status=""
|
||||
ws_last_url=""
|
||||
ws_url_missing_logged=0
|
||||
ws_failed_logged=0
|
||||
while true; do
|
||||
if [ "$(date +%s)" -gt "$deadline" ]; then
|
||||
ws_last_err=$(tenant_call GET "/workspaces/$wid" 2>/dev/null | \
|
||||
python3 -c "import json,sys; print(json.load(sys.stdin).get('last_sample_error',''))" 2>/dev/null || echo "")
|
||||
fail "Workspace $wid never reached online with a routable URL within ${WORKSPACE_ONLINE_TIMEOUT_SECS}s (~$((WORKSPACE_ONLINE_TIMEOUT_SECS/60)) min) (last status=$ws_last_status, url=$ws_last_url, err=$ws_last_err)"
|
||||
fi
|
||||
ws_json=$(tenant_call GET "/workspaces/$wid" 2>/dev/null || echo '{}')
|
||||
ws_status=$(echo "$ws_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('status') or '')" 2>/dev/null)
|
||||
ws_url=$(echo "$ws_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('url') or '')" 2>/dev/null)
|
||||
if [ "$ws_status" != "$ws_last_status" ]; then
|
||||
log " $wid → $ws_status"
|
||||
ws_last_status="$ws_status"
|
||||
fi
|
||||
if [ -n "$ws_url" ] && [ "$ws_url" != "$ws_last_url" ]; then
|
||||
log " $wid url ready: $ws_url"
|
||||
ws_last_url="$ws_url"
|
||||
fi
|
||||
case "$ws_status" in
|
||||
online)
|
||||
if [ -n "$ws_url" ]; then
|
||||
break
|
||||
fi
|
||||
if [ "$ws_url_missing_logged" = "0" ]; then
|
||||
log " $wid online but URL is not assigned yet — waiting for workspace routing readiness"
|
||||
ws_url_missing_logged=1
|
||||
fi
|
||||
sleep 10
|
||||
;;
|
||||
failed)
|
||||
# Not a hard fail — bootstrap-watcher frequently marks failed at
|
||||
# 5 min on hermes, then heartbeat recovers to online around 10-13
|
||||
# min when install.sh finishes. Log once per workspace so the CI
|
||||
# output isn't spammy.
|
||||
if [ "$ws_failed_logged" = "0" ]; then
|
||||
log " $wid transiently failed — waiting for heartbeat recovery (bootstrap-watcher deadline, see cp#245)"
|
||||
ws_failed_logged=1
|
||||
fi
|
||||
sleep 10
|
||||
;;
|
||||
*) sleep 10 ;;
|
||||
esac
|
||||
done
|
||||
ok " $wid online and routable"
|
||||
done
|
||||
}
|
||||
|
||||
# ─── 5. Provision parent workspace ─────────────────────────────────────
|
||||
# Inject the LLM provider key so the runtime can authenticate at boot.
|
||||
# Branch by which secret is set so the script supports multiple paths
|
||||
@@ -383,9 +477,9 @@ elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
# is still independent of MOLECULE_STAGING_OPENAI_API_KEY, so an OpenAI
|
||||
# quota collapse doesn't wedge this path. Pinned to the claude-code
|
||||
# runtime: hermes/langgraph use OpenAI-shaped envs and won't honour
|
||||
# ANTHROPIC_API_KEY without further wiring (out of scope for this
|
||||
# branch; if you need a hermes/Anthropic path, dispatch with
|
||||
# E2E_RUNTIME=hermes + E2E_OPENAI_API_KEY pointing at a working key).
|
||||
# ANTHROPIC_API_KEY without further wiring. pick_model_slug maps this
|
||||
# branch to claude-sonnet-4-6 so the claude-code provider registry
|
||||
# selects anthropic-api instead of the OAuth-only sonnet alias.
|
||||
SECRETS_JSON=$(python3 -c "
|
||||
import json, os
|
||||
k = os.environ['E2E_ANTHROPIC_API_KEY']
|
||||
@@ -410,6 +504,7 @@ print(json.dumps({
|
||||
fi
|
||||
|
||||
MODEL_SLUG=$(pick_model_slug "$RUNTIME")
|
||||
log " MODEL_SLUG=$MODEL_SLUG"
|
||||
|
||||
log "5/11 Provisioning parent workspace (runtime=$RUNTIME)..."
|
||||
PARENT_RESP=$(tenant_call POST /workspaces \
|
||||
@@ -437,48 +532,16 @@ fi
|
||||
# deadline fires at 5 min and sets status=failed prematurely; heartbeat
|
||||
# then transitions failed → online after install.sh finishes. So:
|
||||
#
|
||||
# - 20 min deadline (hermes worst-case + slack)
|
||||
# - ${WORKSPACE_ONLINE_TIMEOUT_SECS}s (~$((WORKSPACE_ONLINE_TIMEOUT_SECS/60)) min)
|
||||
# deadline (hermes worst-case + slack). Configurable via
|
||||
# E2E_WORKSPACE_ONLINE_TIMEOUT_SECS (#1646).
|
||||
# - 'failed' is a TRANSIENT state we must tolerate — log and keep
|
||||
# polling, only hard-fail at the deadline. Pre-bootstrap-watcher-fix
|
||||
# (controlplane#245) this was a flake generator: workspace went
|
||||
# failed→online inside our window but we bailed at the failed read.
|
||||
log "7/11 Waiting for workspace(s) to reach status=online (up to 30 min — hermes cold boot)..."
|
||||
WS_DEADLINE=$(( $(date +%s) + 1800 ))
|
||||
WS_TO_CHECK="$PARENT_ID"
|
||||
[ -n "$CHILD_ID" ] && WS_TO_CHECK="$WS_TO_CHECK $CHILD_ID"
|
||||
for wid in $WS_TO_CHECK; do
|
||||
WS_LAST_STATUS=""
|
||||
WS_FAILED_LOGGED=0
|
||||
while true; do
|
||||
if [ "$(date +%s)" -gt "$WS_DEADLINE" ]; then
|
||||
WS_LAST_ERR=$(tenant_call GET "/workspaces/$wid" 2>/dev/null | \
|
||||
python3 -c "import json,sys; print(json.load(sys.stdin).get('last_sample_error',''))" 2>/dev/null || echo "")
|
||||
fail "Workspace $wid never reached online within 20 min (last status=$WS_LAST_STATUS, err=$WS_LAST_ERR)"
|
||||
fi
|
||||
WS_JSON=$(tenant_call GET "/workspaces/$wid" 2>/dev/null || echo '{}')
|
||||
WS_STATUS=$(echo "$WS_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin).get('status',''))" 2>/dev/null)
|
||||
if [ "$WS_STATUS" != "$WS_LAST_STATUS" ]; then
|
||||
log " $wid → $WS_STATUS"
|
||||
WS_LAST_STATUS="$WS_STATUS"
|
||||
fi
|
||||
case "$WS_STATUS" in
|
||||
online) break ;;
|
||||
failed)
|
||||
# Not a hard fail — bootstrap-watcher frequently marks failed at
|
||||
# 5 min on hermes, then heartbeat recovers to online around 10-13
|
||||
# min when install.sh finishes. Log once per workspace so the CI
|
||||
# output isn't spammy.
|
||||
if [ "$WS_FAILED_LOGGED" = "0" ]; then
|
||||
log " $wid transiently failed — waiting for heartbeat recovery (bootstrap-watcher deadline, see cp#245)"
|
||||
WS_FAILED_LOGGED=1
|
||||
fi
|
||||
sleep 10
|
||||
;;
|
||||
*) sleep 10 ;;
|
||||
esac
|
||||
done
|
||||
ok " $wid online"
|
||||
done
|
||||
WS_TO_CHECK=("$PARENT_ID")
|
||||
[ -n "$CHILD_ID" ] && WS_TO_CHECK+=("$CHILD_ID")
|
||||
wait_workspaces_online_routable "7/11 Waiting for workspace(s) to reach status=online (up to $((WORKSPACE_ONLINE_TIMEOUT_SECS/60)) min — hermes cold boot)..." "${WS_TO_CHECK[@]}"
|
||||
|
||||
# ─── 7b. Canvas-terminal diagnose (EIC chain probe) ────────────────────
|
||||
# This step exists because the canvas-terminal failure of 2026-05-03
|
||||
@@ -490,7 +553,7 @@ done
|
||||
# - tenantIngressRules / workspaceIngressRules (CP)
|
||||
# - eicSSHIngressRule helper (CP)
|
||||
# - AuthorizeIngress source-group support (CP awsapi)
|
||||
# - EIC_ENDPOINT_SG_ID Railway env
|
||||
# - MOLECULE_EIC_ENDPOINT_SG_ID Railway env
|
||||
# - handleRemoteConnect's send-ssh-public-key/open-tunnel/ssh chain
|
||||
# surfaces within ~20 min of merge instead of waiting for a user report.
|
||||
#
|
||||
@@ -504,7 +567,7 @@ done
|
||||
# probes docker.Ping + container exec; we still expect ok=true there
|
||||
# since local-docker is the alternative production path.
|
||||
log "7b/11 Canvas-terminal EIC diagnose probe..."
|
||||
for wid in $WS_TO_CHECK; do
|
||||
for wid in "${WS_TO_CHECK[@]}"; do
|
||||
DIAG_JSON=$(tenant_call GET "/workspaces/$wid/terminal/diagnose" 2>/dev/null || echo '{}')
|
||||
DIAG_OK=$(echo "$DIAG_JSON" | python3 -c "import json,sys; d=json.load(sys.stdin); print('true' if d.get('ok') else 'false')" 2>/dev/null || echo "false")
|
||||
if [ "$DIAG_OK" = "true" ]; then
|
||||
@@ -512,7 +575,7 @@ for wid in $WS_TO_CHECK; do
|
||||
else
|
||||
DIAG_FAIL=$(echo "$DIAG_JSON" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('first_failure','unknown'))" 2>/dev/null || echo "unknown")
|
||||
DIAG_DETAIL=$(echo "$DIAG_JSON" | python3 -c "import json,sys; d=json.load(sys.stdin); s=[x for x in d.get('steps',[]) if not x.get('ok')]; step=s[0] if s else {}; print(' — '.join(x for x in [step.get('error',''), step.get('detail','')] if x))" 2>/dev/null || echo "")
|
||||
fail "Workspace $wid terminal diagnose failed at step '$DIAG_FAIL': $DIAG_DETAIL — check tenant SG has tcp/22 from EIC endpoint SG (sg-0785d5c6138220523), EIC_ENDPOINT_SG_ID set in Railway, and EIC endpoint health"
|
||||
fail "Workspace $wid terminal diagnose failed at step '$DIAG_FAIL': $DIAG_DETAIL — check tenant SG has tcp/22 from the configured EIC endpoint SG, MOLECULE_EIC_ENDPOINT_SG_ID is set in Railway, and EIC endpoint health"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -540,7 +603,7 @@ CONFIG_PAYLOAD="${CONFIG_MARKER}
|
||||
name: synth-canary
|
||||
runtime: ${RUNTIME}
|
||||
"
|
||||
for wid in $WS_TO_CHECK; do
|
||||
for wid in "${WS_TO_CHECK[@]}"; do
|
||||
PUT_BODY=$(python3 -c "import json,sys; print(json.dumps({'content': sys.stdin.read()}))" <<< "$CONFIG_PAYLOAD")
|
||||
# Capture body to a tempfile so curl's -w '%{http_code}' is the only
|
||||
# thing on stdout. The first version used `-w '\n%{http_code}\n'` and
|
||||
@@ -573,6 +636,12 @@ for wid in $WS_TO_CHECK; do
|
||||
ok " $wid config.yaml PUT OK (HTTP $PUT_CODE)"
|
||||
done
|
||||
|
||||
# Saving config.yaml follows the same path as Canvas Config Save & Restart.
|
||||
# The controlplane can briefly put the workspace back into provisioning and
|
||||
# clear its route while the runtime restarts, so A2A must wait on the same
|
||||
# externally routable readiness boundary again.
|
||||
wait_workspaces_online_routable "7d/11 Waiting for workspace(s) to recover routing after config.yaml PUT..." "${WS_TO_CHECK[@]}"
|
||||
|
||||
# ─── 8. A2A round-trip on parent ───────────────────────────────────────
|
||||
log "8/11 Sending A2A message to parent — expecting agent response..."
|
||||
# Smoke prompt phrasing — DO NOT trim back to the bare "Reply with exactly: PONG"
|
||||
@@ -612,10 +681,44 @@ print(json.dumps({
|
||||
# 90s gives ~3x headroom over observed cold-call P95 (~25-30s).
|
||||
# Subsequent A2A turns hit the same workspace and are sub-second, so
|
||||
# this only widens the window for step 8/11 of the canary's first turn.
|
||||
A2A_RESP=$(tenant_call POST "/workspaces/$PARENT_ID/a2a" \
|
||||
--max-time 90 \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$A2A_PAYLOAD")
|
||||
A2A_TMP=$(mktemp -t synth_a2a.XXXXXX)
|
||||
for A2A_ATTEMPT in $(seq 1 12); do
|
||||
: >"$A2A_TMP"
|
||||
set +e
|
||||
A2A_CODE=$(tenant_call POST "/workspaces/$PARENT_ID/a2a" \
|
||||
--max-time 90 \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$A2A_PAYLOAD" \
|
||||
-o "$A2A_TMP" \
|
||||
-w '%{http_code}' \
|
||||
2>/dev/null)
|
||||
A2A_RC=$?
|
||||
set -e
|
||||
A2A_CODE=${A2A_CODE:-000}
|
||||
A2A_RESP=$(cat "$A2A_TMP" 2>/dev/null || echo "")
|
||||
if [ "$A2A_RC" = "0" ] && [ "$A2A_CODE" -ge 200 ] && [ "$A2A_CODE" -lt 300 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
A2A_SAFE_BODY=$(printf '%s' "$A2A_RESP" | sanitize_http_body)
|
||||
if echo "$A2A_CODE" | grep -Eq '^(502|503|504)$' && echo "$A2A_SAFE_BODY" | grep -Eqi 'Service Unavailable|Bad Gateway|Gateway Timeout|error code: 502|error code: 504|workspace agent unreachable|connection refused|no healthy upstream|workspace agent busy|native_session'; then
|
||||
log " A2A cold-start probe attempt $A2A_ATTEMPT/12 returned $A2A_CODE: $A2A_SAFE_BODY"
|
||||
if [ "$A2A_ATTEMPT" -lt 12 ]; then
|
||||
A2A_SLEEP=10
|
||||
if echo "$A2A_SAFE_BODY" | grep -Eqi 'workspace agent busy|native_session'; then
|
||||
A2A_SLEEP=30
|
||||
fi
|
||||
sleep "$A2A_SLEEP"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
break
|
||||
done
|
||||
rm -f "$A2A_TMP"
|
||||
if [ "$A2A_RC" != "0" ] || [ "$A2A_CODE" -lt 200 ] || [ "$A2A_CODE" -ge 300 ]; then
|
||||
A2A_SAFE_BODY=$(printf '%s' "$A2A_RESP" | sanitize_http_body)
|
||||
fail "A2A POST /workspaces/$PARENT_ID/a2a failed after $A2A_ATTEMPT attempt(s) (curl_rc=$A2A_RC, http=$A2A_CODE): $A2A_SAFE_BODY"
|
||||
fi
|
||||
AGENT_TEXT=$(echo "$A2A_RESP" | python3 -c "
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
@@ -812,20 +915,50 @@ print(json.dumps({
|
||||
}
|
||||
}))
|
||||
")
|
||||
set +e
|
||||
# Raw curl (not tenant_call) because this call carries an extra
|
||||
# X-Source-Workspace-Id header. Must still send X-Molecule-Org-Id
|
||||
# or TenantGuard 404s — previously missing, caused section 10 to
|
||||
# fail rc=22 despite everything upstream being correct (2026-04-21).
|
||||
DELEG_RESP=$(curl "${CURL_COMMON[@]}" -X POST "$TENANT_URL/workspaces/$CHILD_ID/a2a" \
|
||||
-H "Authorization: Bearer $EFFECTIVE_TENANT_TOKEN" \
|
||||
-H "X-Molecule-Org-Id: $ORG_ID" \
|
||||
-H "X-Source-Workspace-Id: $PARENT_ID" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$DELEG_PAYLOAD")
|
||||
DELEG_RC=$?
|
||||
set -e
|
||||
[ $DELEG_RC -ne 0 ] && fail "Delegation A2A POST failed (rc=$DELEG_RC)"
|
||||
DELEG_TMP=$(mktemp -t deleg_a2a.XXXXXX)
|
||||
for DELEG_ATTEMPT in $(seq 1 12); do
|
||||
: >"$DELEG_TMP"
|
||||
set +e
|
||||
# Raw curl (not tenant_call) because this call carries an extra
|
||||
# X-Source-Workspace-Id header. Must still send X-Molecule-Org-Id
|
||||
# or TenantGuard 404s — previously missing, caused section 10 to
|
||||
# fail rc=22 despite everything upstream being correct (2026-04-21).
|
||||
DELEG_CODE=$(curl "${CURL_COMMON[@]}" -X POST "$TENANT_URL/workspaces/$CHILD_ID/a2a" \
|
||||
-H "Authorization: Bearer $EFFECTIVE_TENANT_TOKEN" \
|
||||
-H "X-Molecule-Org-Id: $ORG_ID" \
|
||||
-H "X-Source-Workspace-Id: $PARENT_ID" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$DELEG_PAYLOAD" \
|
||||
-o "$DELEG_TMP" \
|
||||
-w '%{http_code}' \
|
||||
2>/dev/null)
|
||||
DELEG_RC=$?
|
||||
set -e
|
||||
DELEG_CODE=${DELEG_CODE:-000}
|
||||
DELEG_RESP=$(cat "$DELEG_TMP" 2>/dev/null || echo "")
|
||||
if [ "$DELEG_RC" = "0" ] && [ "$DELEG_CODE" -ge 200 ] && [ "$DELEG_CODE" -lt 300 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
DELEG_SAFE_BODY=$(printf '%s' "$DELEG_RESP" | sanitize_http_body)
|
||||
if echo "$DELEG_CODE" | grep -Eq '^(502|503|504)$' && echo "$DELEG_SAFE_BODY" | grep -Eqi 'Service Unavailable|Bad Gateway|Gateway Timeout|error code: 502|error code: 504|workspace agent unreachable|connection refused|no healthy upstream|workspace agent busy|native_session'; then
|
||||
log " Delegation A2A cold-start attempt $DELEG_ATTEMPT/12 returned $DELEG_CODE: $DELEG_SAFE_BODY"
|
||||
if [ "$DELEG_ATTEMPT" -lt 12 ]; then
|
||||
DELEG_SLEEP=10
|
||||
if echo "$DELEG_SAFE_BODY" | grep -Eqi 'workspace agent busy|native_session'; then
|
||||
DELEG_SLEEP=30
|
||||
fi
|
||||
sleep "$DELEG_SLEEP"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
break
|
||||
done
|
||||
rm -f "$DELEG_TMP"
|
||||
if [ "$DELEG_RC" != "0" ] || [ "$DELEG_CODE" -lt 200 ] || [ "$DELEG_CODE" -ge 300 ]; then
|
||||
DELEG_SAFE_BODY=$(printf '%s' "$DELEG_RESP" | sanitize_http_body)
|
||||
fail "Delegation A2A POST failed after $DELEG_ATTEMPT attempt(s) (curl_rc=$DELEG_RC, http=$DELEG_CODE): $DELEG_SAFE_BODY"
|
||||
fi
|
||||
DELEG_TEXT=$(echo "$DELEG_RESP" | python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_staging_e2e_workflows_use_stable_minimax_default() -> None:
|
||||
"""Keep cron/push E2E on the same MiniMax model as the smoke-tested script."""
|
||||
workflow_paths = [
|
||||
".gitea/workflows/e2e-staging-saas.yml",
|
||||
".gitea/workflows/staging-smoke.yml",
|
||||
".gitea/workflows/continuous-synth-e2e.yml",
|
||||
]
|
||||
|
||||
for rel in workflow_paths:
|
||||
text = (ROOT / rel).read_text()
|
||||
assert "MiniMax-M2.7-highspeed" not in text
|
||||
assert "MiniMax-M2" in text
|
||||
@@ -705,7 +705,7 @@ def test_ci_change_detector_docs_and_meta_scripts_do_not_trigger_surfaces():
|
||||
}
|
||||
|
||||
|
||||
def test_ci_platform_go_pr_steps_are_path_scoped():
|
||||
def test_ci_platform_go_steps_are_path_scoped_on_all_events():
|
||||
doc = yaml.safe_load(CI_WORKFLOW.read_text(encoding="utf-8"))
|
||||
platform = doc["jobs"]["platform-build"]
|
||||
assert platform.get("needs") == "changes"
|
||||
@@ -720,11 +720,11 @@ def test_ci_platform_go_pr_steps_are_path_scoped():
|
||||
assert expensive_steps
|
||||
for step in expensive_steps:
|
||||
expr = step.get("if", "")
|
||||
assert "github.event_name != 'pull_request'" in expr
|
||||
assert "needs.changes.outputs.platform == 'true'" in expr
|
||||
assert "github.event_name != 'pull_request'" not in expr
|
||||
|
||||
|
||||
def test_ci_canvas_nextjs_pr_steps_are_path_scoped():
|
||||
def test_ci_canvas_nextjs_steps_are_path_scoped_on_all_events():
|
||||
doc = yaml.safe_load(CI_WORKFLOW.read_text(encoding="utf-8"))
|
||||
canvas = doc["jobs"]["canvas-build"]
|
||||
assert canvas.get("needs") == "changes"
|
||||
@@ -739,11 +739,11 @@ def test_ci_canvas_nextjs_pr_steps_are_path_scoped():
|
||||
assert expensive_steps
|
||||
for step in expensive_steps:
|
||||
expr = step.get("if", "")
|
||||
assert "github.event_name != 'pull_request'" in expr
|
||||
assert "needs.changes.outputs.canvas == 'true'" in expr
|
||||
assert "github.event_name != 'pull_request'" not in expr
|
||||
|
||||
|
||||
def test_ci_shellcheck_pr_steps_are_path_scoped():
|
||||
def test_ci_shellcheck_steps_are_path_scoped_on_all_events():
|
||||
doc = yaml.safe_load(CI_WORKFLOW.read_text(encoding="utf-8"))
|
||||
shellcheck = doc["jobs"]["shellcheck"]
|
||||
assert shellcheck.get("needs") == "changes"
|
||||
@@ -756,5 +756,5 @@ def test_ci_shellcheck_pr_steps_are_path_scoped():
|
||||
assert expensive_steps
|
||||
for step in expensive_steps:
|
||||
expr = step.get("if", "")
|
||||
assert "github.event_name != 'pull_request'" in expr
|
||||
assert "needs.changes.outputs.scripts == 'true'" in expr
|
||||
assert "github.event_name != 'pull_request'" not in expr
|
||||
|
||||
@@ -159,7 +159,8 @@ func generateAppInstallationToken() (string, time.Time, error) {
|
||||
req, _ := http.NewRequest("POST", fmt.Sprintf("https://api.github.com/app/installations/%d/access_tokens", installID), nil)
|
||||
req.Header.Set("Authorization", "Bearer "+signed)
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestWorkspaceCreate_WithParentID(t *testing.T) {
|
||||
// Default tier is 3 (Privileged) — see workspace.go create-handler comment.
|
||||
// delivery_mode defaults to "push" when payload omits it (#2339).
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Child Agent", nil, 3, "langgraph", sqlmock.AnyArg(), &parentID, nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Child Agent", nil, 3, "langgraph", sqlmock.AnyArg(), &parentID, nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
@@ -69,7 +69,7 @@ func TestWorkspaceCreate_ExplicitClaudeCodeRuntime(t *testing.T) {
|
||||
mock.ExpectBegin()
|
||||
// delivery_mode defaults to "push" when payload omits it (#2339).
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "CC Agent", nil, 2, "claude-code", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "CC Agent", nil, 2, "claude-code", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
@@ -291,7 +291,7 @@ func TestWorkspaceCreate_MaxConcurrentTasksOverride(t *testing.T) {
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Leader Agent", nil, 3, "claude-code", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), 3, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Leader Agent", nil, 3, "claude-code", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), 3, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
|
||||
@@ -368,7 +368,7 @@ func TestWorkspaceCreate(t *testing.T) {
|
||||
// Default tier is 3 (Privileged) — see workspace.go create-handler comment.
|
||||
// delivery_mode defaults to "push" when payload omits it (#2339).
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Test Agent", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Test Agent", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
// Expect transaction commit (no secrets in this payload)
|
||||
|
||||
@@ -214,6 +214,11 @@ func (h *WorkspaceHandler) Create(c *gin.Context) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid workspace fields"})
|
||||
return
|
||||
}
|
||||
// #1686 Phase 1: validate per-workspace compute overrides.
|
||||
if err := models.ValidateComputeConfig(payload.Compute); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
id := uuid.New().String()
|
||||
awarenessNamespace := workspaceAwarenessNamespace(id)
|
||||
@@ -398,11 +403,22 @@ func (h *WorkspaceHandler) Create(c *gin.Context) {
|
||||
// double-click. Helper retries with " (2)", " (3)", … up to maxNameSuffix,
|
||||
// returns the actually-persisted name (which we MUST thread back into
|
||||
// payload + broadcast so the canvas displays what the DB has).
|
||||
var computeInstanceType *string
|
||||
var computeVolumeRootGB *int
|
||||
if payload.Compute != nil {
|
||||
if payload.Compute.InstanceType != "" {
|
||||
computeInstanceType = &payload.Compute.InstanceType
|
||||
}
|
||||
if payload.Compute.Volume.RootGB != 0 {
|
||||
computeVolumeRootGB = &payload.Compute.Volume.RootGB
|
||||
}
|
||||
}
|
||||
|
||||
const insertWorkspaceSQL = `
|
||||
INSERT INTO workspaces (id, name, role, tier, runtime, awareness_namespace, status, parent_id, workspace_dir, workspace_access, budget_limit, max_concurrent_tasks, delivery_mode)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'provisioning', $7, $8, $9, $10, $11, $12)
|
||||
INSERT INTO workspaces (id, name, role, tier, runtime, awareness_namespace, status, parent_id, workspace_dir, workspace_access, budget_limit, max_concurrent_tasks, delivery_mode, compute_instance_type, compute_volume_root_gb)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'provisioning', $7, $8, $9, $10, $11, $12, $13, $14)
|
||||
`
|
||||
insertArgs := []any{id, payload.Name, role, payload.Tier, payload.Runtime, awarenessNamespace, payload.ParentID, workspaceDir, workspaceAccess, payload.BudgetLimit, maxConcurrent, deliveryMode}
|
||||
insertArgs := []any{id, payload.Name, role, payload.Tier, payload.Runtime, awarenessNamespace, payload.ParentID, workspaceDir, workspaceAccess, payload.BudgetLimit, maxConcurrent, deliveryMode, computeInstanceType, computeVolumeRootGB}
|
||||
persistedName, currentTx, err := insertWorkspaceWithNameRetry(
|
||||
ctx,
|
||||
tx,
|
||||
|
||||
@@ -157,6 +157,8 @@ func TestWorkspaceBudget_Create_WithLimit(t *testing.T) {
|
||||
&budgetVal, // budget_limit ($10)
|
||||
models.DefaultMaxConcurrentTasks, // max_concurrent_tasks default
|
||||
"push", // delivery_mode default (#2339)
|
||||
(*string)(nil), // compute_instance_type default
|
||||
(*int)(nil), // compute_volume_root_gb default
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
@@ -309,9 +309,31 @@ func (h *WorkspaceHandler) buildProvisionerConfig(
|
||||
// RuntimeImages[Runtime] :latest lookup, which is what the dead
|
||||
// reader's sql.ErrNoRows path was producing already.
|
||||
Image: "",
|
||||
// Compute overrides (nullable — omitted = platform-managed default).
|
||||
// Issue #1686 Phase 1.
|
||||
InstanceType: extractComputeInstanceType(payload.Compute),
|
||||
VolumeRootGB: extractComputeVolumeRootGB(payload.Compute),
|
||||
}
|
||||
}
|
||||
|
||||
// extractComputeInstanceType returns the instance type from a ComputeConfig,
|
||||
// or nil when cfg is nil or the field is empty.
|
||||
func extractComputeInstanceType(cfg *models.ComputeConfig) *string {
|
||||
if cfg != nil && cfg.InstanceType != "" {
|
||||
return &cfg.InstanceType
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractComputeVolumeRootGB returns the root volume size from a ComputeConfig,
|
||||
// or nil when cfg is nil or the field is zero.
|
||||
func extractComputeVolumeRootGB(cfg *models.ComputeConfig) *int {
|
||||
if cfg != nil && cfg.Volume.RootGB != 0 {
|
||||
return &cfg.Volume.RootGB
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// issueAndInjectToken rotates the workspace auth token and injects the
|
||||
// plaintext into cfg.ConfigFiles[".auth_token"] so it is written into the
|
||||
// /configs volume by WriteFilesToContainer immediately after the container
|
||||
|
||||
@@ -779,6 +779,75 @@ func TestBuildProvisionerConfig_WorkspacePathFromEnv(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildProvisionerConfig_ComputeOverrides verifies that #1686 Phase 1
|
||||
// compute fields (instance_type + volume.root_gb) are threaded from the
|
||||
// create payload into the provisioner config.
|
||||
func TestBuildProvisionerConfig_ComputeOverrides(t *testing.T) {
|
||||
mock := setupTestDB(t)
|
||||
mock.ExpectQuery(`SELECT COALESCE\(workspace_dir`).
|
||||
WithArgs("ws-compute").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"workspace_dir", "workspace_access"}).AddRow("", "none"))
|
||||
|
||||
broadcaster := newTestBroadcaster()
|
||||
handler := NewWorkspaceHandler(broadcaster, nil, "http://localhost:8080", t.TempDir())
|
||||
|
||||
cfg := handler.buildProvisionerConfig(
|
||||
context.Background(),
|
||||
"ws-compute",
|
||||
"",
|
||||
nil,
|
||||
models.CreateWorkspacePayload{
|
||||
Tier: 2,
|
||||
Runtime: "python",
|
||||
Compute: &models.ComputeConfig{
|
||||
InstanceType: "g4dn.xlarge",
|
||||
Volume: models.ComputeVolume{RootGB: 256},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
"",
|
||||
"workspace:ws-compute",
|
||||
)
|
||||
|
||||
if cfg.InstanceType == nil || *cfg.InstanceType != "g4dn.xlarge" {
|
||||
t.Errorf("InstanceType = %v, want g4dn.xlarge", cfg.InstanceType)
|
||||
}
|
||||
if cfg.VolumeRootGB == nil || *cfg.VolumeRootGB != 256 {
|
||||
t.Errorf("VolumeRootGB = %v, want 256", cfg.VolumeRootGB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildProvisionerConfig_ComputeNil verifies backward compat: when the
|
||||
// payload omits compute, the provisioner config fields are nil so the CP
|
||||
// applies its own defaults.
|
||||
func TestBuildProvisionerConfig_ComputeNil(t *testing.T) {
|
||||
mock := setupTestDB(t)
|
||||
mock.ExpectQuery(`SELECT COALESCE\(workspace_dir`).
|
||||
WithArgs("ws-no-compute").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"workspace_dir", "workspace_access"}).AddRow("", "none"))
|
||||
|
||||
broadcaster := newTestBroadcaster()
|
||||
handler := NewWorkspaceHandler(broadcaster, nil, "http://localhost:8080", t.TempDir())
|
||||
|
||||
cfg := handler.buildProvisionerConfig(
|
||||
context.Background(),
|
||||
"ws-no-compute",
|
||||
"",
|
||||
nil,
|
||||
models.CreateWorkspacePayload{Tier: 1, Runtime: "python"},
|
||||
nil,
|
||||
"",
|
||||
"workspace:ws-no-compute",
|
||||
)
|
||||
|
||||
if cfg.InstanceType != nil {
|
||||
t.Errorf("InstanceType = %v, want nil", cfg.InstanceType)
|
||||
}
|
||||
if cfg.VolumeRootGB != nil {
|
||||
t.Errorf("VolumeRootGB = %v, want nil", cfg.VolumeRootGB)
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== issueAndInjectToken (issue #418) ====================
|
||||
|
||||
// TestIssueAndInjectToken_HappyPath verifies that on a normal (re)provision the
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -342,7 +343,7 @@ func TestWorkspaceCreate_DBInsertError(t *testing.T) {
|
||||
// Transaction begins, workspace INSERT fails, transaction is rolled back.
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Failing Agent", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Failing Agent", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnError(sql.ErrConnDone)
|
||||
mock.ExpectRollback()
|
||||
|
||||
@@ -364,6 +365,94 @@ func TestWorkspaceCreate_DBInsertError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestWorkspaceCreate_InvalidCompute verifies #1686 Phase 1 create-time
|
||||
// validation: bad instance_type or volume.root_gb returns 400 before any
|
||||
// DB call.
|
||||
func TestWorkspaceCreate_InvalidCompute(t *testing.T) {
|
||||
broadcaster := newTestBroadcaster()
|
||||
handler := NewWorkspaceHandler(broadcaster, nil, "http://localhost:8080", t.TempDir())
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
body string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "instance_type too long",
|
||||
body: `{"name":"Bad Type","compute":{"instance_type":"` + strings.Repeat("x", 65) + `"}}`,
|
||||
want: "compute.instance_type too long",
|
||||
},
|
||||
{
|
||||
name: "root_gb too small",
|
||||
body: `{"name":"Small Disk","compute":{"volume":{"root_gb":16}}}`,
|
||||
want: "compute.volume.root_gb must be at least 32",
|
||||
},
|
||||
{
|
||||
name: "root_gb too large",
|
||||
body: `{"name":"Big Disk","compute":{"volume":{"root_gb":4096}}}`,
|
||||
want: "compute.volume.root_gb exceeds maximum 2048",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Request = httptest.NewRequest("POST", "/workspaces", bytes.NewBufferString(tc.body))
|
||||
c.Request.Header.Set("Content-Type", "application/json")
|
||||
|
||||
handler.Create(c)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), tc.want) {
|
||||
t.Errorf("body %q should contain %q", w.Body.String(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWorkspaceCreate_WithComputeOverrides verifies that valid #1686 Phase 1
|
||||
// compute fields are persisted into the workspaces table.
|
||||
func TestWorkspaceCreate_WithComputeOverrides(t *testing.T) {
|
||||
mock := setupTestDB(t)
|
||||
setupTestRedis(t)
|
||||
broadcaster := newTestBroadcaster()
|
||||
handler := NewWorkspaceHandler(broadcaster, nil, "http://localhost:8080", t.TempDir())
|
||||
|
||||
mock.ExpectBegin()
|
||||
instanceType := "g4dn.xlarge"
|
||||
rootGB := 256
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "GPU Agent", nil, 3, "python", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", &instanceType, &rootGB).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("INSERT INTO structure_events").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("INSERT INTO structure_events").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`UPDATE workspaces SET status =`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("INSERT INTO workspace_config").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
body := `{"name":"GPU Agent","runtime":"python","compute":{"instance_type":"g4dn.xlarge","volume":{"root_gb":256}}}`
|
||||
c.Request = httptest.NewRequest("POST", "/workspaces", bytes.NewBufferString(body))
|
||||
c.Request.Header.Set("Content-Type", "application/json")
|
||||
|
||||
handler.Create(c)
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Errorf("expected 201, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Errorf("unmet sqlmock expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceCreate_DefaultsApplied(t *testing.T) {
|
||||
mock := setupTestDB(t)
|
||||
setupTestRedis(t)
|
||||
@@ -375,7 +464,7 @@ func TestWorkspaceCreate_DefaultsApplied(t *testing.T) {
|
||||
// Expect workspace INSERT with defaulted tier=3 (Privileged — the
|
||||
// handler default in workspace.go), runtime="langgraph"
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Default Agent", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Default Agent", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
@@ -423,7 +512,7 @@ func TestWorkspaceCreate_SaaSHardForcesTier4(t *testing.T) {
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "SaaS External Agent", nil, 4, "external", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "SaaS External Agent", nil, 4, "external", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
@@ -464,7 +553,7 @@ func TestWorkspaceCreate_WithSecrets_Persists(t *testing.T) {
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Hermes Agent", nil, 3, "hermes", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Hermes Agent", nil, 3, "hermes", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
// Secret inserted inside the same transaction.
|
||||
mock.ExpectExec("INSERT INTO workspace_secrets").
|
||||
@@ -576,7 +665,7 @@ func TestWorkspaceCreate_ExternalURL_SSRFSafe(t *testing.T) {
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Ext Agent", nil, 3, "external", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Ext Agent", nil, 3, "external", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
// External URL update (localhost is explicitly allowed by validateAgentURL).
|
||||
@@ -615,7 +704,7 @@ func TestWorkspaceCreate_KimiRuntime_PreservesLabel(t *testing.T) {
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Kimi Agent", nil, 3, "kimi", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Kimi Agent", nil, 3, "kimi", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
// Pre-register flow: awaiting_agent + runtime preserved as "kimi"
|
||||
@@ -1639,7 +1728,7 @@ runtime_config:
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(
|
||||
sqlmock.AnyArg(), "Hermes Agent", nil, 3, "hermes",
|
||||
sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
@@ -1696,7 +1785,7 @@ model: anthropic:claude-sonnet-4-5
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(
|
||||
sqlmock.AnyArg(), "Legacy Agent", nil, 3, "langgraph",
|
||||
sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
@@ -1749,7 +1838,7 @@ runtime_config:
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(
|
||||
sqlmock.AnyArg(), "Custom Hermes", nil, 3, "hermes",
|
||||
sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
@@ -1855,7 +1944,7 @@ func TestWorkspaceCreate_188_NoTemplateNoRuntime_StillDefaultsLanggraph(t *testi
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Plain Default", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Plain Default", nil, 3, "langgraph", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
@@ -1890,7 +1979,7 @@ func TestWorkspaceCreate_188_ExplicitRuntimeNoTemplate_OK(t *testing.T) {
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT INTO workspaces").
|
||||
WithArgs(sqlmock.AnyArg(), "Explicit Codex", nil, 3, "codex", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push").
|
||||
WithArgs(sqlmock.AnyArg(), "Explicit Codex", nil, 3, "codex", sqlmock.AnyArg(), (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push", (*string)(nil), (*int)(nil)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectExec("INSERT INTO canvas_layouts").
|
||||
|
||||
@@ -3,6 +3,7 @@ package models
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -45,6 +46,10 @@ type Workspace struct {
|
||||
// forced to route updates through a parent workspace. Default true
|
||||
// (preserves existing behaviour for all workspaces).
|
||||
TalkToUserEnabled bool `json:"talk_to_user_enabled" db:"talk_to_user_enabled"`
|
||||
// Compute overrides (nullable — omitted = platform-managed default).
|
||||
// Issue #1686 Phase 1.
|
||||
ComputeInstanceType *string `json:"compute_instance_type,omitempty" db:"compute_instance_type"`
|
||||
ComputeVolumeRootGB *int `json:"compute_volume_root_gb,omitempty" db:"compute_volume_root_gb"`
|
||||
// Canvas layout fields (from JOIN)
|
||||
X float64 `json:"x"`
|
||||
Y float64 `json:"y"`
|
||||
@@ -154,6 +159,40 @@ type MemorySeed struct {
|
||||
Scope string `json:"scope" yaml:"scope"` // LOCAL, TEAM, GLOBAL
|
||||
}
|
||||
|
||||
// ComputeVolume holds per-workspace disk configuration.
|
||||
type ComputeVolume struct {
|
||||
RootGB int `json:"root_gb"`
|
||||
}
|
||||
|
||||
// ComputeConfig holds per-workspace EC2 compute overrides.
|
||||
// Omitted at create time means "use platform-managed defaults".
|
||||
type ComputeConfig struct {
|
||||
InstanceType string `json:"instance_type"`
|
||||
Volume ComputeVolume `json:"volume"`
|
||||
}
|
||||
|
||||
// ValidateComputeConfig performs create-time validation on compute overrides.
|
||||
// Returns nil when cfg is nil (omitted = platform-managed default).
|
||||
func ValidateComputeConfig(cfg *ComputeConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
if cfg.InstanceType != "" {
|
||||
if len(cfg.InstanceType) > 64 {
|
||||
return fmt.Errorf("compute.instance_type too long (max 64 chars)")
|
||||
}
|
||||
}
|
||||
if cfg.Volume.RootGB != 0 {
|
||||
if cfg.Volume.RootGB < 32 {
|
||||
return fmt.Errorf("compute.volume.root_gb must be at least 32")
|
||||
}
|
||||
if cfg.Volume.RootGB > 2048 {
|
||||
return fmt.Errorf("compute.volume.root_gb exceeds maximum 2048")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type CreateWorkspacePayload struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Role string `json:"role"`
|
||||
@@ -180,6 +219,9 @@ type CreateWorkspacePayload struct {
|
||||
// MaxConcurrentTasks caps parallel A2A + cron dispatch. 0 means use
|
||||
// DefaultMaxConcurrentTasks. Leaders typically set 3.
|
||||
MaxConcurrentTasks int `json:"max_concurrent_tasks"`
|
||||
// Compute is an optional per-workspace EC2 shape override.
|
||||
// Omitted = platform-managed default (current behaviour).
|
||||
Compute *ComputeConfig `json:"compute,omitempty"`
|
||||
Canvas struct {
|
||||
X float64 `json:"x"`
|
||||
Y float64 `json:"y"`
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package models
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidateComputeConfig_NilIsValid(t *testing.T) {
|
||||
if err := ValidateComputeConfig(nil); err != nil {
|
||||
t.Errorf("nil compute config should be valid, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateComputeConfig_EmptyIsValid(t *testing.T) {
|
||||
cfg := &ComputeConfig{}
|
||||
if err := ValidateComputeConfig(cfg); err != nil {
|
||||
t.Errorf("empty compute config should be valid, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateComputeConfig_ValidOverrides(t *testing.T) {
|
||||
cfg := &ComputeConfig{
|
||||
InstanceType: "g4dn.xlarge",
|
||||
Volume: ComputeVolume{RootGB: 256},
|
||||
}
|
||||
if err := ValidateComputeConfig(cfg); err != nil {
|
||||
t.Errorf("valid overrides should pass, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateComputeConfig_InstanceTypeTooLong(t *testing.T) {
|
||||
longName := string(make([]byte, 65))
|
||||
for i := range longName {
|
||||
longName = longName[:i] + "x" + longName[i+1:]
|
||||
}
|
||||
cfg := &ComputeConfig{InstanceType: longName}
|
||||
if err := ValidateComputeConfig(cfg); err == nil {
|
||||
t.Error("expected error for instance_type > 64 chars")
|
||||
} else if err.Error() != "compute.instance_type too long (max 64 chars)" {
|
||||
t.Errorf("unexpected error message: %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateComputeConfig_RootGBTooSmall(t *testing.T) {
|
||||
cfg := &ComputeConfig{Volume: ComputeVolume{RootGB: 31}}
|
||||
if err := ValidateComputeConfig(cfg); err == nil {
|
||||
t.Error("expected error for root_gb < 32")
|
||||
} else if err.Error() != "compute.volume.root_gb must be at least 32" {
|
||||
t.Errorf("unexpected error message: %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateComputeConfig_RootGBTooLarge(t *testing.T) {
|
||||
cfg := &ComputeConfig{Volume: ComputeVolume{RootGB: 2049}}
|
||||
if err := ValidateComputeConfig(cfg); err == nil {
|
||||
t.Error("expected error for root_gb > 2048")
|
||||
} else if err.Error() != "compute.volume.root_gb exceeds maximum 2048" {
|
||||
t.Errorf("unexpected error message: %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateComputeConfig_BoundaryValues(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
cfg ComputeConfig
|
||||
ok bool
|
||||
}{
|
||||
{"min root_gb", ComputeConfig{Volume: ComputeVolume{RootGB: 32}}, true},
|
||||
{"max root_gb", ComputeConfig{Volume: ComputeVolume{RootGB: 2048}}, true},
|
||||
{"just under min", ComputeConfig{Volume: ComputeVolume{RootGB: 31}}, false},
|
||||
{"just over max", ComputeConfig{Volume: ComputeVolume{RootGB: 2049}}, false},
|
||||
{"exactly 64 char type", ComputeConfig{InstanceType: string(make([]byte, 64))}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// fill the 64-char case with 'x'
|
||||
if tc.cfg.InstanceType != "" {
|
||||
b := make([]byte, len(tc.cfg.InstanceType))
|
||||
for i := range b {
|
||||
b[i] = 'x'
|
||||
}
|
||||
tc.cfg.InstanceType = string(b)
|
||||
}
|
||||
err := ValidateComputeConfig(&tc.cfg)
|
||||
if tc.ok && err != nil {
|
||||
t.Errorf("expected valid, got: %v", err)
|
||||
}
|
||||
if !tc.ok && err == nil {
|
||||
t.Error("expected invalid, got nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -163,6 +163,10 @@ type cpProvisionRequest struct {
|
||||
// collectCPConfigFiles which rejects symlinks and non-regular files
|
||||
// before including them. Serialised as base64 to avoid JSON escaping.
|
||||
ConfigFiles map[string]string `json:"config_files,omitempty"`
|
||||
// Compute overrides (nullable — omitted = platform-managed default).
|
||||
// Issue #1686 Phase 1.
|
||||
InstanceType *string `json:"instance_type,omitempty"`
|
||||
VolumeRootGB *int `json:"volume_root_gb,omitempty"`
|
||||
}
|
||||
|
||||
type cpProvisionResponse struct {
|
||||
@@ -206,13 +210,15 @@ func (p *CPProvisioner) Start(ctx context.Context, cfg WorkspaceConfig) (string,
|
||||
}
|
||||
|
||||
req := cpProvisionRequest{
|
||||
OrgID: p.orgID,
|
||||
WorkspaceID: cfg.WorkspaceID,
|
||||
Runtime: cfg.Runtime,
|
||||
Tier: cfg.Tier,
|
||||
PlatformURL: cfg.PlatformURL,
|
||||
Env: env,
|
||||
ConfigFiles: configFiles,
|
||||
OrgID: p.orgID,
|
||||
WorkspaceID: cfg.WorkspaceID,
|
||||
Runtime: cfg.Runtime,
|
||||
Tier: cfg.Tier,
|
||||
PlatformURL: cfg.PlatformURL,
|
||||
Env: env,
|
||||
ConfigFiles: configFiles,
|
||||
InstanceType: cfg.InstanceType,
|
||||
VolumeRootGB: cfg.VolumeRootGB,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(req)
|
||||
|
||||
@@ -1062,3 +1062,75 @@ func TestCollectCPConfigFiles_RejectsRootSymlink(t *testing.T) {
|
||||
t.Errorf("expected symlink-related error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStart_ComputeOverrides — when WorkspaceConfig carries InstanceType and
|
||||
// VolumeRootGB, they must be forwarded in the cpProvisionRequest body so the
|
||||
// CP can pass them to EC2 RunInstances. Regression guard for #1686 Phase 1.
|
||||
func TestStart_ComputeOverrides(t *testing.T) {
|
||||
var gotBody cpProvisionRequest
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
|
||||
t.Errorf("decode request: %v", err)
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = io.WriteString(w, `{"instance_id":"i-compute","state":"pending"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := &CPProvisioner{baseURL: srv.URL, orgID: "org-1", httpClient: srv.Client()}
|
||||
instanceType := "g4dn.xlarge"
|
||||
volumeRootGB := 256
|
||||
_, err := p.Start(context.Background(), WorkspaceConfig{
|
||||
WorkspaceID: "ws-1",
|
||||
Runtime: "python",
|
||||
Tier: 2,
|
||||
PlatformURL: "http://tenant",
|
||||
InstanceType: &instanceType,
|
||||
VolumeRootGB: &volumeRootGB,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
if gotBody.InstanceType == nil || *gotBody.InstanceType != "g4dn.xlarge" {
|
||||
t.Errorf("instance_type = %v, want g4dn.xlarge", gotBody.InstanceType)
|
||||
}
|
||||
if gotBody.VolumeRootGB == nil || *gotBody.VolumeRootGB != 256 {
|
||||
t.Errorf("volume_root_gb = %v, want 256", gotBody.VolumeRootGB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStart_ComputeOmittedWhenNil — when WorkspaceConfig has no compute
|
||||
// overrides, the JSON body must omit the keys entirely (omitempty) so CP
|
||||
// applies its own defaults rather than empty/zero values.
|
||||
func TestStart_ComputeOmittedWhenNil(t *testing.T) {
|
||||
var raw json.RawMessage
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
t.Errorf("decode request: %v", err)
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = io.WriteString(w, `{"instance_id":"i-default","state":"pending"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := &CPProvisioner{baseURL: srv.URL, orgID: "org-1", httpClient: srv.Client()}
|
||||
_, err := p.Start(context.Background(), WorkspaceConfig{
|
||||
WorkspaceID: "ws-1",
|
||||
Runtime: "python",
|
||||
Tier: 1,
|
||||
PlatformURL: "http://tenant",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
var decoded map[string]interface{}
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal raw body: %v", err)
|
||||
}
|
||||
if _, ok := decoded["instance_type"]; ok {
|
||||
t.Errorf("instance_type should be omitted when nil")
|
||||
}
|
||||
if _, ok := decoded["volume_root_gb"]; ok {
|
||||
t.Errorf("volume_root_gb should be omitted when nil")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,6 +105,11 @@ type WorkspaceConfig struct {
|
||||
WorkspaceAccess string // #65: "none" (default), "read_only", or "read_write"
|
||||
ResetClaudeSession bool // #12: if true, discard the claude-sessions volume before start (fresh session dir)
|
||||
|
||||
// Compute overrides (nullable — omitted = platform-managed default).
|
||||
// Issue #1686 Phase 1.
|
||||
InstanceType *string `json:"instance_type,omitempty"`
|
||||
VolumeRootGB *int `json:"volume_root_gb,omitempty"`
|
||||
|
||||
// Image, when non-empty, overrides the runtime→image lookup. CP
|
||||
// (molecule-controlplane) is the single SSOT for runtime image digest
|
||||
// pins via its migrations/027_runtime_image_pins table — the pin is
|
||||
@@ -726,6 +731,16 @@ func buildContainerEnv(cfg WorkspaceConfig) []string {
|
||||
}
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
// #1687: alias GH_PAT → GH_TOKEN / GITHUB_TOKEN on the READ side
|
||||
// (container env assembly). gh CLI and git credential helpers look
|
||||
// for these standard names; by aliasing here we avoid writing the
|
||||
// forbidden keys into tenant-writer surfaces (workspace_secrets,
|
||||
// envVars map, etc.). GH_PAT itself is not an SCM-write credential
|
||||
// and passes through cfg.EnvVars untouched.
|
||||
if pat, hasPAT := cfg.EnvVars["GH_PAT"]; hasPAT && pat != "" {
|
||||
env = append(env, fmt.Sprintf("GH_TOKEN=%s", pat))
|
||||
env = append(env, fmt.Sprintf("GITHUB_TOKEN=%s", pat))
|
||||
}
|
||||
// Inject ADMIN_TOKEN from the platform server's environment so workspace
|
||||
// containers can call /admin/liveness and other admin-gated endpoints
|
||||
// (core#831). cp_provisioner.go handles this separately for SaaS tenants.
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# T4 privilege contract — generated from
|
||||
# molecule-ai/molecule-core workspace-server/internal/provisioner/t4_privilege_contract.go
|
||||
# RFC: molecule-ai/internal#456
|
||||
# Do NOT edit this file by hand; regenerate via `go run ./cmd/t4-contract-dump > t4_capabilities.yaml`.
|
||||
version: 1
|
||||
agent_uid: 1000
|
||||
capabilities:
|
||||
- name: "agent_home_writable"
|
||||
description: "/agent-home is writable by the agent (Files API split per task #128). The Files API redesign uses /agent-home as the user-writable root; the agent must be able to create files there without sudo."
|
||||
severity: hard
|
||||
source: "task #128 Files API redesign; memory reference_post_suspension_pipeline"
|
||||
probe: "TF=/agent-home/.t4-cap-write-probe-${MOLECULE_T4_PROBE_ID:-$$}; echo ok > \"$TF\" && [ \"$(cat \"$TF\")\" = \"ok\" ] && rm -f \"$TF\""
|
||||
- name: "agent_uid_1000"
|
||||
description: "The container's primary process (the runtime, post-gosu) runs as uid 1000, not root. T4 grants full machine access via privileged + host PID + Docker socket — the WORKLOAD inside that privileged container must still be unprivileged to prevent every untrusted code execution from being trivially root-on-host."
|
||||
severity: hard
|
||||
source: "RFC internal#456 §2.1.2; memory feedback_hermes_listpeers_401_token_root600_unreadable_by_uid1000"
|
||||
probe: "[ \"$(id -u)\" = \"1000\" ]"
|
||||
- name: "auth_token_agent_owned"
|
||||
description: "/configs/.auth_token is owned by uid 1000 (== AgentUID) so the a2a_mcp_server can read its bearer. In SaaS mode molecule-runtime itself writes the token via save_token() — the ownership equals the runtime's exec uid. If the runtime ever runs as root, this fails and list_peers 401s (the Hermes class bug)."
|
||||
severity: hard
|
||||
source: "RFC internal#456 §10; memory feedback_hermes_listpeers_401_token_root600_unreadable_by_uid1000"
|
||||
probe: "[ -e /configs/.auth_token ] && [ \"$(stat -c '%u' /configs/.auth_token)\" = \"1000\" ]"
|
||||
- name: "docker_socket_reachable"
|
||||
description: "/var/run/docker.sock is bind-mounted and host Docker is reachable from the T4 container. The probe enters the host mount+PID namespaces before running docker info so it validates the same host-control path production agents use, instead of depending on the template image's Docker CLI/socket group details."
|
||||
severity: hard
|
||||
source: "provisioner.go applyHostConfig T4 branch (case 4)"
|
||||
probe: "sudo -n nsenter --target 1 --mount --pid -- docker info >/dev/null 2>&1"
|
||||
- name: "host_fs_write_readback"
|
||||
description: "Host filesystem is mounted at /host and the agent can write+read+remove a file there via sudo. Proves real host reach (not just a PID-1 namespace trick on an isolated init)."
|
||||
severity: hard
|
||||
source: "RFC internal#456 §11"
|
||||
probe: "MARKER=\"t4cap-$(date +%s)-$RANDOM\"; PROBE_FILE=\"/host/tmp/.t4-cap-probe-${MOLECULE_T4_PROBE_ID:-$$}\"; sudo -n sh -c \"echo $MARKER > $PROBE_FILE\" && [ \"$(sudo -n cat $PROBE_FILE)\" = \"$MARKER\" ] && sudo -n rm -f $PROBE_FILE"
|
||||
- name: "host_root_reach_via_nsenter"
|
||||
description: "The uid-1000 agent can attain host root via `sudo -n nsenter --target 1 --mount --pid -- id -u` returning 0. This is the T4 escalation leg: full machine access means the agent CAN escalate to host root deliberately, even though it does not run as root by default."
|
||||
severity: hard
|
||||
source: "RFC internal#456 §11; memory reference_per_template_privilege_contract_class_audit_2026_05_16"
|
||||
probe: "[ \"$(sudo -n nsenter --target 1 --mount --pid -- id -u)\" = \"0\" ]"
|
||||
- name: "list_peers_http_200"
|
||||
description: "The platform list_peers HTTP endpoint (served by the in-container a2a_mcp_server) returns HTTP 200 when called from uid 1000 with the bearer from /configs/.auth_token. This proves the WHOLE token-ownership chain end-to-end: token written under correct uid → reader uid matches → bearer non-empty → platform accepts. A self-contained empirical test for the Hermes class bug."
|
||||
severity: hard
|
||||
source: "memory reference_openclaw_fresh_provision_nonfunctional_anthropic_default_unroutable; memory reference_openclaw_mcp_peer_wiring_rootcause"
|
||||
probe: "BEARER=$(cat /configs/.auth_token 2>/dev/null || echo \"\"); [ -n \"$BEARER\" ] || exit 1; PORT=$(cat /configs/.platform_port 2>/dev/null || echo \"8080\"); STATUS=$(curl -sS -o /dev/null -w '%{http_code}' -H \"Authorization: Bearer $BEARER\" \"http://127.0.0.1:${PORT}/list_peers\"); [ \"$STATUS\" = \"200\" ]"
|
||||
- name: "network_egress_https"
|
||||
description: "Generic HTTPS egress works. T4 is unconstrained network; the canonical test target is the Molecule-owned Gitea middleman over its public name. CI must not depend on GitHub or other mirrors for this probe. Any reachable HTTPS endpoint satisfies it — the YAML carries the recommended targets but accepts any 200/301/302."
|
||||
severity: hard
|
||||
source: "task #174 brief"
|
||||
probe: "for U in $MOLECULE_T4_EGRESS_TARGETS; do C=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 8 \"$U\"); case \"$C\" in 2*|3*) exit 0;; esac; done; exit 1"
|
||||
required_egress:
|
||||
- "https://git.moleculesai.app/api/v1/version"
|
||||
- name: "pid_host_visible"
|
||||
description: "Host PID namespace is shared (--pid=host). The container can see host process 1 (systemd or pid-1 on the EC2 instance). Required for nsenter into host mount/pid namespaces."
|
||||
severity: hard
|
||||
source: "provisioner.go applyHostConfig T4 branch (case 4): hostCfg.PidMode = 'host'"
|
||||
probe: "[ \"$(sudo -n nsenter --target 1 --mount --pid -- id -u)\" = \"0\" ]"
|
||||
- name: "privileged_flag_observable"
|
||||
description: "Container is started with --privileged. Observable from inside via /proc/self/status CapEff containing CAP_SYS_ADMIN. Defense-in-depth for the provisioner emission side."
|
||||
severity: advisory
|
||||
source: "provisioner.go applyHostConfig T4 branch (case 4)"
|
||||
probe: "grep -q '^CapEff:.*ffffffffff' /proc/self/status"
|
||||
@@ -120,8 +120,8 @@ func T4PrivilegeContract() []T4Capability {
|
||||
},
|
||||
{
|
||||
Name: "docker_socket_reachable",
|
||||
Description: "/var/run/docker.sock is bind-mounted into the container so the agent can manage other containers (T4 use case: agent-as-orchestrator). Proven by 'docker version' returning a server section, which requires the daemon to answer over the socket.",
|
||||
Probe: `sudo -n docker version --format '{{.Server.Version}}' >/dev/null 2>&1`,
|
||||
Description: "/var/run/docker.sock is bind-mounted and host Docker is reachable from the T4 container. The probe enters the host mount+PID namespaces before running docker info so it validates the same host-control path production agents use, instead of depending on the template image's Docker CLI/socket group details.",
|
||||
Probe: `sudo -n nsenter --target 1 --mount --pid -- docker info >/dev/null 2>&1`,
|
||||
Severity: SeverityHard,
|
||||
Source: "provisioner.go applyHostConfig T4 branch (case 4)",
|
||||
},
|
||||
@@ -145,7 +145,7 @@ func T4PrivilegeContract() []T4Capability {
|
||||
},
|
||||
{
|
||||
Name: "network_egress_https",
|
||||
Description: "Generic HTTPS egress works. T4 is unconstrained network; the canonical test target is the Gitea instance over its public name, which any fork user can also resolve. Any reachable HTTPS endpoint satisfies it — the YAML carries the recommended targets but accepts any 200/301/302.",
|
||||
Description: "Generic HTTPS egress works. T4 is unconstrained network; the canonical test target is the Molecule-owned Gitea middleman over its public name. CI must not depend on GitHub or other mirrors for this probe. Any reachable HTTPS endpoint satisfies it — the YAML carries the recommended targets but accepts any 200/301/302.",
|
||||
Probe: `for U in $MOLECULE_T4_EGRESS_TARGETS; do ` +
|
||||
` C=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 8 "$U"); ` +
|
||||
` case "$C" in 2*|3*) exit 0;; esac; ` +
|
||||
@@ -153,10 +153,9 @@ func T4PrivilegeContract() []T4Capability {
|
||||
Severity: SeverityHard,
|
||||
Source: "task #174 brief",
|
||||
RequiredEgress: []string{
|
||||
// Public, no auth, returns a small JSON.
|
||||
// Molecule-owned, public, no auth, returns a small JSON.
|
||||
// Adopters override via MOLECULE_T4_EGRESS_TARGETS.
|
||||
"https://api.github.com/zen",
|
||||
"https://www.google.com/generate_204",
|
||||
"https://git.moleculesai.app/api/v1/version",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -169,7 +168,7 @@ func T4PrivilegeContract() []T4Capability {
|
||||
{
|
||||
Name: "pid_host_visible",
|
||||
Description: "Host PID namespace is shared (--pid=host). The container can see host process 1 (systemd or pid-1 on the EC2 instance). Required for nsenter into host mount/pid namespaces.",
|
||||
Probe: `[ -d /proc/1/root ] && [ "$(sudo -n readlink /proc/1/ns/pid)" = "$(sudo -n readlink /proc/self/ns/pid)" ]`,
|
||||
Probe: `[ "$(sudo -n nsenter --target 1 --mount --pid -- id -u)" = "0" ]`,
|
||||
Severity: SeverityHard,
|
||||
Source: "provisioner.go applyHostConfig T4 branch (case 4): hostCfg.PidMode = 'host'",
|
||||
},
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package provisioner
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -77,6 +78,19 @@ func TestT4PrivilegeContract_CoreCapabilitiesPresent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestT4PrivilegeContract_DefaultEgressUsesMoleculeOwnedEndpoint(t *testing.T) {
|
||||
for _, c := range T4PrivilegeContract() {
|
||||
for _, target := range c.RequiredEgress {
|
||||
if strings.Contains(target, "github.com") {
|
||||
t.Errorf("capability %q default egress target must not depend on GitHub mirror/API: %s", c.Name, target)
|
||||
}
|
||||
if strings.Contains(target, "google.com") {
|
||||
t.Errorf("capability %q default egress target must not depend on external Google endpoint: %s", c.Name, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestT4PrivilegeContract_HardCapabilitiesMajority sanity-checks that
|
||||
// the contract is not silently advisory-only. If someone marks
|
||||
// everything as "advisory" the gate becomes a no-op without anyone
|
||||
@@ -142,6 +156,17 @@ func TestAsYAML_EscapesEmbeddedQuotes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedT4CapabilitiesYAMLMatchesSSOT(t *testing.T) {
|
||||
got, err := os.ReadFile("t4_capabilities.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("read generated t4_capabilities.yaml: %v", err)
|
||||
}
|
||||
want := AsYAML(T4PrivilegeContract())
|
||||
if string(got) != want {
|
||||
t.Fatal("generated t4_capabilities.yaml drifted from T4PrivilegeContract; regenerate with `go run ./cmd/t4-contract-dump > internal/provisioner/t4_capabilities.yaml`")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAgentUIDConsistency ties the contract to the existing
|
||||
// provisioner-side AgentUID const. The probe for "agent_uid_1000"
|
||||
// hard-codes `id -u == 1000`; if AgentUID ever changes (no one
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE workspaces
|
||||
DROP COLUMN IF EXISTS compute_instance_type;
|
||||
|
||||
ALTER TABLE workspaces
|
||||
DROP COLUMN IF EXISTS compute_volume_root_gb;
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Per-workspace EC2 compute configuration (#1686 Phase 1).
|
||||
-- Allows callers to override instance_type and root volume size
|
||||
-- at workspace creation time. Omitted/null values preserve the
|
||||
-- platform-managed default (current behaviour), so this is fully
|
||||
-- backwards-compatible.
|
||||
ALTER TABLE workspaces
|
||||
ADD COLUMN IF NOT EXISTS compute_instance_type TEXT;
|
||||
|
||||
ALTER TABLE workspaces
|
||||
ADD COLUMN IF NOT EXISTS compute_volume_root_gb INTEGER;
|
||||
Reference in New Issue
Block a user