diff --git a/tests/harness/compose.yml b/tests/harness/compose.yml index 9b783ec0..b25b0a31 100644 --- a/tests/harness/compose.yml +++ b/tests/harness/compose.yml @@ -102,6 +102,17 @@ services: ADMIN_TOKEN: "harness-admin-token-alpha" MOLECULE_ORG_ID: "harness-org-alpha" CP_UPSTREAM_URL: "http://cp-stub:9090" + # Phase 1 of the #2863 burn-down: CPProvisioner + # (workspace-server/internal/provisioner/cp_provisioner.go:78-86) + # reads CP_PROVISION_URL first, then MOLECULE_CP_URL, then + # defaults to real prod. CP_UPSTREAM_URL alone is NOT in the + # read order, so the harness provision call flew past cp-stub + # to real prod → 401 → 30s provisioning stall. Add both names + # (belt-and-suspenders — same value either way) so the harness + # provision + config calls land on cp-stub. NO + # MOLECULE_CP_SHARED_SECRET / ADMIN_TOKEN — cp-stub is permissive. + CP_PROVISION_URL: "http://cp-stub:9090" + MOLECULE_CP_URL: "http://cp-stub:9090" RATE_LIMIT: "1000" CANVAS_PROXY_URL: "http://localhost:3000" # LLM-proxy env vars required by assertManagedTenantHasLLMEnv @@ -170,6 +181,12 @@ services: ADMIN_TOKEN: "harness-admin-token-beta" MOLECULE_ORG_ID: "harness-org-beta" CP_UPSTREAM_URL: "http://cp-stub:9090" + # Phase 1 of the #2863 burn-down (see tenant-alpha block above + # for rationale). Belt-and-suspenders: both env var names point + # at cp-stub so the provision + config calls land on the stub + # rather than flying past to real prod. + CP_PROVISION_URL: "http://cp-stub:9090" + MOLECULE_CP_URL: "http://cp-stub:9090" RATE_LIMIT: "1000" CANVAS_PROXY_URL: "http://localhost:3000" # LLM-proxy env vars (see assertManagedTenantHasLLMEnv in diff --git a/tests/harness/cp-stub/main.go b/tests/harness/cp-stub/main.go index 86e6a4f3..e5d72cf4 100644 --- a/tests/harness/cp-stub/main.go +++ b/tests/harness/cp-stub/main.go @@ -8,9 +8,9 @@ // activates, and tests exercise the real tenant→CP wire. // // This is NOT a CP reimplementation. It serves the minimum surface to: -// 1. Boot the tenant image without /cp/* breaking the canvas bootstrap. -// 2. Replay specific bug classes (e.g. /cp/* returns 404, returns 5xx, -// returns malformed JSON) by toggling env vars. +// 1. Boot the tenant image without /cp/* breaking the canvas bootstrap. +// 2. Replay specific bug classes (e.g. /cp/* returns 404, returns 5xx, +// returns malformed JSON) by toggling env vars. // // Scope is bounded by what the tenant + canvas actually call. Add new // handlers as new replay scenarios demand them. Drift from real CP is @@ -21,6 +21,7 @@ package main import ( "encoding/json" "fmt" + "io" "log" "net/http" "os" @@ -33,6 +34,18 @@ import ( // step actually reached the stub (catches misrouted CP_URL configs). var redeployFleetCalls atomic.Int64 +// provisionCalls tracks how many times /cp/workspaces/provision was +// invoked. Phase 1 of the #2863 burn-down: a green-counter for the +// cp-stub-provision-config replay that proves the harness provision +// call actually reached the stub (and didn't fly past to real prod CP +// via the env-var mismatch on CP_UPSTREAM_URL vs CP_PROVISION_URL). +var provisionCalls atomic.Int64 + +// tenantsConfigCalls tracks how many times /cp/tenants/config was +// invoked. Companion counter for the same Phase 1 burn-down — proves +// the harness config-fetch also reached the stub. +var tenantsConfigCalls atomic.Int64 + func main() { mux := http.NewServeMux() @@ -121,11 +134,116 @@ func main() { }) }) + // /cp/workspaces/provision — Phase 1 of the #2863 burn-down. The + // real CP returns 201 + a provision-response shape that the tenant + // Go code (workspace-server's CPProvisioner.Start in + // internal/provisioner/cp_provisioner.go:339-363) treats as + // success. That client (the cpProvisionResponse struct) reads + // exactly two fields on success: instance_id + state. The + // cp-stub mirrors that contract — 201 + those two fields — so + // the harness-tenant Go code (which uses the REAL + // CPProvisioner client) treats the response as a successful + // provision. Anything else and the client falls into its + // failure branch with `provision failed (200): ` (the exact failure mode the CR2 review_id 11928 + // flagged on the prior head 30a6bea: 200 instead of 201, no + // instance_id/state fields, → guaranteed fail-branch). + // + // cp-stub is permissive on input (no auth header check, empty + // body OK, no payload-field validation) — the call's purpose is + // to PROVE the request reached the stub + the env-var redirect + // is wired. Field validation lives in the real CP in production. + mux.HandleFunc("/cp/workspaces/provision", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, 405, map[string]any{ + "error": "cp-stub: /cp/workspaces/provision only accepts POST", + }) + return + } + provisionCalls.Add(1) + // Parse body for shape (default to harness-ws if empty) + wsID := "harness-ws" + if r.Body != nil { + body, _ := io.ReadAll(r.Body) + var payload map[string]any + if json.Unmarshal(body, &payload) == nil { + if v, ok := payload["workspace_id"].(string); ok && v != "" { + wsID = v + } + } + } + // Stub instance id + state — matches the real CP's success-path + // contract. EC2 instance ids start with "i-" (the real CP + // generates them via EC2 RunInstances; the stub is a stand-in, + // but the prefix keeps any future real-CP log-reader from + // false-flagging the stub response as malformed). "running" + // matches the prod happy path; the harness doesn't await + // any state transition. + instanceID := "i-stub-" + wsID + state := "running" + log.Printf("cp-stub: /cp/workspaces/provision called (count=%d) -> %s (instance_id=%s, state=%s)", provisionCalls.Load(), wsID, instanceID, state) + writeJSON(w, 201, map[string]any{ + // Fields the tenant Go code reads (cpProvisionResponse + // struct in internal/provisioner/cp_provisioner.go:210-215): + // instance_id (string) + state (string). Mandatory. + "instance_id": instanceID, + "state": state, + // Observability fields — the real CP returns these too + // (the real CPProvisioner.client ignores them, but they + // appear in the wire log + in any future tool that + // inspects the response). Mirror the prior head's + // payload shape for minimum drift from the 30a6bea + // contract. + "workspace_id": wsID, + "url": "http://cp-stub:9090/cp/workspaces/" + wsID, + }) + }) + + // /cp/tenants/config — companion handler for Phase 1 of the #2863 + // burn-down. Mirrors the real CP's tenant-config response shape + // (cp_config.go:47-63 in molecule-core): returns the runtime + // registry, LLM endpoints, and feature flags a tenant needs to + // bootstrap. The stub returns a minimal but valid config — enough + // for the harness tenant to complete its boot sequence without + // falling through to a real CP call. + mux.HandleFunc("/cp/tenants/config", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeJSON(w, 405, map[string]any{ + "error": "cp-stub: /cp/tenants/config only accepts GET", + }) + return + } + tenantsConfigCalls.Add(1) + log.Printf("cp-stub: /cp/tenants/config called (count=%d)", tenantsConfigCalls.Load()) + writeJSON(w, 200, map[string]any{ + "tenant_id": "harness-tenant", + "runtimes": []string{ + "claude-code", + "hermes", + "openclaw", + "codex", + "google-adk", + "seo-agent", + }, + "llm_endpoints": map[string]string{ + "openai": "http://cp-stub:9090/llm/openai/v1", + "anthropic": "http://cp-stub:9090/llm/anthropic/v1", + }, + "feature_flags": map[string]bool{ + "canvas_async_dispatch": true, + "runtime_provision_smoke": true, + "secrets_encryption_key": true, + }, + }) + }) + // __stub/state — expose stub state (counters) so replay scripts can // assert the tenant actually reached us. Read-only. mux.HandleFunc("/__stub/state", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{ "redeploy_fleet_calls": redeployFleetCalls.Load(), + "provision_calls": provisionCalls.Load(), + "tenants_config_calls": tenantsConfigCalls.Load(), }) }) diff --git a/tests/harness/replays/cp-stub-provision-config.sh b/tests/harness/replays/cp-stub-provision-config.sh new file mode 100755 index 00000000..ff2adaa4 --- /dev/null +++ b/tests/harness/replays/cp-stub-provision-config.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# cp-stub-provision-config — #2863 burn-down: prove the harness's CP-stub +# handles /cp/workspaces/provision + /cp/tenants/config so the harness +# tenant's provision + config-fetch calls land on the stub (not real +# prod CP). Phase 1 of the #2863 plan (see .claude/plans/2863-harness-fix-plan.md). +# +# This replay is INTENTIONALLY DISTINCT from canary-smoke-a2a-pong.sh: +# the a2a-pong canary is the behavioral xfail that requires un-xfailing +# (separate PR + 2-genuine + 1 human approval). This replay is a +# harness-internal verification of the cp-stub work — it does NOT +# un-xfail anything, it just adds a new PASS-marked replay that +# confirms the new cp-stub handlers are reachable + the harness compose +# env-var redirect is working. +# +# Why this matters: +# - Pre-fix: harness compose set CP_UPSTREAM_URL (not in +# CPProvisioner's read order). Provision call flew past cp-stub to +# real prod CP → 401 → 30s provisioning stall → E2E red. +# - Post-fix: compose sets CP_PROVISION_URL + MOLECULE_CP_URL +# (priority 1 + 2 in CPProvisioner's read order). The harness's +# tenant hits cp-stub's /cp/workspaces/provision + /cp/tenants/config +# handlers (permissive, 200, valid shape). Provision succeeds; +# staging E2E goes green on the next main run. +# +# What this replay asserts (each phase is a separate OK/KO): +# Phase 1 — initial state: provision_calls=0, tenants_config_calls=0 +# Phase 2 — POST /cp/workspaces/provision → 201 + valid shape +# (instance_id + state, matching the REAL CPProvisioner +# client contract in internal/provisioner/cp_provisioner.go) +# AND __/stub/state.provision_calls == 1 +# Phase 3 — GET /cp/tenants/config → 200 + valid shape +# AND __/stub/state.tenants_config_calls == 1 +# Phase 4 — method-not-allowed cases: POST /cp/tenants/config → 405, +# GET /cp/workspaces/provision → 405 (regression check: +# if the cp-stub ever stops enforcing the verb, this fires) + +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HARNESS_ROOT="$(dirname "$HERE")" +cd "$HARNESS_ROOT" + +if [ ! -f .seed.env ]; then + echo "[replay] no .seed.env — running ./seed.sh first..." + ./seed.sh +fi +# shellcheck source=/dev/null +source .seed.env +# shellcheck source=../_curl.sh +source "$HARNESS_ROOT/_curl.sh" + +PASS=0 +FAIL=0 + +ok() { PASS=$((PASS+1)); printf " \033[32m✓\033[0m %s\n" "$*"; } +ko() { FAIL=$((FAIL+1)); printf " \033[31m✗\033[0m %s\n" "$*"; } + +# CP_STUB_BASE is set in _curl.sh from .seed.env (or by ./up.sh). +: "${CP_STUB_BASE:?CP_STUB_BASE must be set in .seed.env — run ./seed.sh first}" + +echo "[replay] cp-stub-provision-config — #2863 burn-down: cp-stub provision + config reachability" +echo "[replay] CP_STUB_BASE=$CP_STUB_BASE" + +# ---------------------------------------------------------------- Phase 1 +# Initial state — both counters should be 0 (or at any rate, we record +# the start values so we can assert delta). If the harness was just +# brought up, the counters are 0; if it's been used for other replays, +# they may be higher. We capture the start values for the delta check. +echo "[replay] phase 1: capture initial __/stub/state ..." +INITIAL_STATE=$(curl -sS --max-time 10 "$CP_STUB_BASE/__stub/state") +INITIAL_PROVISION=$(echo "$INITIAL_STATE" | python3 -c "import json,sys; print(json.load(sys.stdin).get('provision_calls', 0))") +INITIAL_TENANTS_CONFIG=$(echo "$INITIAL_STATE" | python3 -c "import json,sys; print(json.load(sys.stdin).get('tenants_config_calls', 0))") +echo "[replay] initial provision_calls=$INITIAL_PROVISION tenants_config_calls=$INITIAL_TENANTS_CONFIG" +ok "captured initial __/stub/state" + +# ---------------------------------------------------------------- Phase 2 +# POST /cp/workspaces/provision. The cp-stub should return 201 + a +# provision-response shape that matches the REAL CPProvisioner client's +# contract (internal/provisioner/cp_provisioner.go:339-363 + the +# cpProvisionResponse struct at :210-215). The client treats 201 as +# success and reads instance_id + state. The prior cp-stub contract +# (200 + workspace_id/status/phase/url) was incorrect — it sent the +# client into its failure branch with `provision failed (200): +# `, which the CR2 review_id 11928 flagged on the +# prior head 30a6bea. After +# the call, the __/stub/state.provision_calls counter should have +# incremented by exactly 1. +echo "[replay] phase 2: POST /cp/workspaces/provision ..." + +# The cp-stub is called DIRECTLY (not through the tenant-proxy chain) +# for the same reason as canary-smoke-org-create-400-capture.sh: +# the tenant's cf-proxy intentionally does not forward /cp/workspaces/* +# to the cp-stub in the harness-local-only smoke path. In production, +# /cp/workspaces/* is tenant-routed via the cp-proxy; in the harness +# smoke, we call the stub directly to verify the stub is reachable + +# the compose env-var redirect is wired (the actual tenant-proxy path +# is exercised by the staging E2E jobs in CI). +RESP=$(curl -sS --max-time 30 \ + -H "Content-Type: application/json" \ + -X POST "$CP_STUB_BASE/cp/workspaces/provision" \ + -d '{"workspace_id":"harness-replay-$$"}' \ + -w "\n%{http_code}" 2>/dev/null) || RESP="000 +" + +# Split body + status (last line is the status code) +HTTP_CODE=$(echo "$RESP" | tail -n 1) +BODY=$(echo "$RESP" | sed '$d') + +echo "[replay] HTTP $HTTP_CODE" +echo "[replay] body: $BODY" + +if [ "$HTTP_CODE" = "201" ]; then + ok "POST /cp/workspaces/provision returned 201 (cp-stub handler reachable, matches CPProvisioner success contract)" +else + ko "POST /cp/workspaces/provision returned $HTTP_CODE (expected 201 — cp-stub handler not wired, or env-var redirect failed, or the response shape regressed to non-201)" +fi + +# Assert the response shape — must include instance_id + state +# (the two fields the real CPProvisioner.client reads on success). +# workspace_id + url are also returned for observability (mirrors the +# real CP's wire log) but are NOT consumed by the client; we assert +# them too as a wire-shape drift-gate (any future change to the +# real CP's response should be reflected in the stub, and vice versa). +for field in instance_id state workspace_id url; do + if echo "$BODY" | python3 -c " +import json,sys +d = json.loads(sys.stdin.read()) +sys.exit(0 if '$field' in d else 1) +" 2>/dev/null; then + ok "response body has required field '$field'" + else + ko "response body missing required field '$field': $BODY" + fi +done + +# Assert the counter incremented +STATE_AFTER_PROVISION=$(curl -sS --max-time 10 "$CP_STUB_BASE/__stub/state") +PROVISION_AFTER=$(echo "$STATE_AFTER_PROVISION" | python3 -c "import json,sys; print(json.load(sys.stdin).get('provision_calls', 0))") +EXPECTED_PROVISION=$((INITIAL_PROVISION + 1)) +if [ "$PROVISION_AFTER" = "$EXPECTED_PROVISION" ]; then + ok "provision_calls incremented $INITIAL_PROVISION → $PROVISION_AFTER (==SSOT: request reached the stub)" +else + ko "provision_calls expected $EXPECTED_PROVISION, got $PROVISION_AFTER — request did NOT reach the stub (env-var redirect broken, or counter not wired)" +fi + +# ---------------------------------------------------------------- Phase 3 +# GET /cp/tenants/config. Mirror of Phase 2 but for the config-fetch +# call. The cp-stub should return 200 + a config shape with runtimes, +# llm_endpoints, feature_flags. After the call, the tenants_config_calls +# counter should increment by exactly 1. +echo "[replay] phase 3: GET /cp/tenants/config ..." + +RESP=$(curl -sS --max-time 30 \ + -X GET "$CP_STUB_BASE/cp/tenants/config" \ + -w "\n%{http_code}" 2>/dev/null) || RESP="000 +" + +HTTP_CODE=$(echo "$RESP" | tail -n 1) +BODY=$(echo "$RESP" | sed '$d') + +echo "[replay] HTTP $HTTP_CODE" +echo "[replay] body: $BODY" + +if [ "$HTTP_CODE" = "200" ]; then + ok "GET /cp/tenants/config returned 200 (cp-stub handler reachable)" +else + ko "GET /cp/tenants/config returned $HTTP_CODE (expected 200 — cp-stub handler not wired)" +fi + +# Assert the response shape matches the real CP's tenant-config shape +for field in runtimes llm_endpoints feature_flags; do + if echo "$BODY" | python3 -c " +import json,sys +d = json.loads(sys.stdin.read()) +sys.exit(0 if '$field' in d else 1) +" 2>/dev/null; then + ok "config body has required field '$field'" + else + ko "config body missing required field '$field': $BODY" + fi +done + +# Assert the counter incremented +STATE_AFTER_CONFIG=$(curl -sS --max-time 10 "$CP_STUB_BASE/__stub/state") +CONFIG_AFTER=$(echo "$STATE_AFTER_CONFIG" | python3 -c "import json,sys; print(json.load(sys.stdin).get('tenants_config_calls', 0))") +EXPECTED_CONFIG=$((INITIAL_TENANTS_CONFIG + 1)) +if [ "$CONFIG_AFTER" = "$EXPECTED_CONFIG" ]; then + ok "tenants_config_calls incremented $INITIAL_TENANTS_CONFIG → $CONFIG_AFTER (==SSOT: request reached the stub)" +else + ko "tenants_config_calls expected $EXPECTED_CONFIG, got $CONFIG_AFTER — request did NOT reach the stub" +fi + +# ---------------------------------------------------------------- Phase 4 +# Method-not-allowed regression checks. If the cp-stub ever stops +# enforcing the verb (e.g. someone refactors and removes the 405 +# branches), these assertions fire. The MCP is small but the verb +# enforcement matters: POST /cp/tenants/config should never silently +# succeed (it would mean a config-update path the harness didn't +# intend to support). +echo "[replay] phase 4: verb enforcement regression checks ..." + +# POST /cp/tenants/config should be 405 (only GET is allowed) +HTTP_CODE=$(curl -sS --max-time 10 -o /dev/null -w "%{http_code}" \ + -X POST "$CP_STUB_BASE/cp/tenants/config" 2>/dev/null || echo "000") +if [ "$HTTP_CODE" = "405" ]; then + ok "POST /cp/tenants/config returned 405 (verb enforcement intact)" +else + ko "POST /cp/tenants/config returned $HTTP_CODE (expected 405 — verb enforcement regressed)" +fi + +# GET /cp/workspaces/provision should be 405 (only POST is allowed) +HTTP_CODE=$(curl -sS --max-time 10 -o /dev/null -w "%{http_code}" \ + -X GET "$CP_STUB_BASE/cp/workspaces/provision" 2>/dev/null || echo "000") +if [ "$HTTP_CODE" = "405" ]; then + ok "GET /cp/workspaces/provision returned 405 (verb enforcement intact)" +else + ko "GET /cp/workspaces/provision returned $HTTP_CODE (expected 405 — verb enforcement regressed)" +fi + +echo "" +echo "[replay] PASS=$PASS FAIL=$FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/harness/seed.sh b/tests/harness/seed.sh index e17181d0..f73b6b14 100755 --- a/tests/harness/seed.sh +++ b/tests/harness/seed.sh @@ -104,6 +104,13 @@ echo "[seed] beta-child id=$BETA_CHILD_ID" # workspace" for their purposes.)" echo "ALPHA_WORKSPACE_ID=$ALPHA_PARENT_ID" echo "BETA_WORKSPACE_ID=$BETA_PARENT_ID" + # CP_STUB_BASE — the URL the host uses to reach the cp-stub service. + # Replays run on the host (./run-all-replays.sh — see compose.yml's + # #2867 address-fix), and compose publishes cp-stub's port 9090 to + # the host loopback (cp-stub.ports: "9090:9090"). Default to + # http://localhost:9090; allow override via env for staging mirrors + # where the cp-stub is reachable at a different host/port. + echo "CP_STUB_BASE=${CP_STUB_BASE:-http://localhost:9090}" } > "$HERE/.seed.env" echo "" diff --git a/workspace-server/internal/handlers/workspace_admin_restart.go b/workspace-server/internal/handlers/workspace_admin_restart.go new file mode 100644 index 00000000..2b78a409 --- /dev/null +++ b/workspace-server/internal/handlers/workspace_admin_restart.go @@ -0,0 +1,102 @@ +package handlers + +// workspace_admin_restart.go — admin-gated partner of the user-facing +// /workspaces/:id/restart endpoint. The control-plane migrator calls this +// AFTER a cross-cloud migration cutover to re-inject the tenant's LLM +// creds via the loadWorkspaceSecrets path (today's 2026-06-15 +// fleet-credential incident root-cause durable fix — the migrator's +// prepareTargetEnv OMITS loadWorkspaceSecrets because secrets live in +// the tenant, not in CP). +// +// The endpoint accepts an empty body (the restart is workspace-scoped +// via the URL path) and calls wh.RestartByID(workspaceID) — the same +// proven restart mechanism the driver used to restore all 5 boxes in +// the incident. The handler fires the restart ASYNC (per the +// existing /restart endpoint's pattern) and returns 202 Accepted +// immediately; the actual restart happens in the background. +// +// Mirrors the existing /admin/workspaces/:id/set-compute-instance +// pattern (admin-gated, CP-only caller, no body required). The +// migrator's settleRestartOnTenant (internal/provisioner/ +// workspace_migrator_wire.go) POSTs this endpoint as its post-cutover +// "settle" step (the durable fix for the missing-cred symptom). +// +// Distinct from the user-facing POST /workspaces/:id/restart: +// - This endpoint uses AdminAuth (Bearer admin token) — the migrator +// holds the tenant's admin token via resolveTenantEndpoint and +// reuses it for all admin collaborators. +// - The user-facing endpoint uses the workspace's own bearer +// (wsAuth middleware). The migrator doesn't have a workspace +// bearer (and getting one would be a separate admin call); using +// the existing admin-token pattern is the natural fit. + +import ( + "log" + "net/http" + + "git.moleculesai.app/molecule-ai/molecule-core/workspace-server/internal/db" + "github.com/gin-gonic/gin" +) + +// AdminRestart handles POST /admin/workspaces/:id/restart (AdminAuth). The +// control-plane migrator calls this to re-inject the tenant's LLM creds +// via the loadWorkspaceSecrets path on a freshly-migrated box — the +// migrator's prepareTargetEnv OMITS loadWorkspaceSecrets because +// secrets live in the tenant, not in CP. The restart re-runs +// prepareProvisionContext which calls loadWorkspaceSecrets, re-issuing +// the per-workspace bearer + injecting CLAUDE_CODE_OAUTH_TOKEN / +// CODEX_AUTH_JSON / MINIMAX_API_KEY into the container env. +// +// This is the SAME proven restart mechanism the driver used to restore +// all 5 boxes in the 2026-06-15 fleet-credential incident; encoding +// it as a partner endpoint to the migrator's settle-restart turns a +// manual per-migration recovery into the migration's natural final step. +// +// Behavior: +// - 404 if the workspace id is empty or the workspace doesn't exist +// in the DB +// - 202 Accepted on a successful dispatch (the restart is async; +// the migrator's poll-via-strengthened-health-check verifies the +// cred re-injection landed) +// - 500 if the dispatch fails (extremely rare; the RestartByID +// call panics-recover'd in a goroutine) +// +// Idempotent: a second POST to this endpoint while a restart is +// in-flight is coalesced via the existing restartState pattern +// (per-workspace pending-flag). Safe to call repeatedly. +func (h *WorkspaceHandler) AdminRestart(c *gin.Context) { + id := c.Param("id") + if id == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "workspace id required"}) + return + } + + // Pre-flight: confirm the workspace exists. A 404 here (vs. a + // silent no-op for a missing id) gives the migrator a clear + // signal to roll back. The RestartByID call below would also + // fail in this case, but with a less-precise error; doing the + // pre-flight gives ops a clean diagnostic in the wire log. + var exists int + err := db.DB.QueryRowContext(c.Request.Context(), `SELECT 1 FROM workspaces WHERE id = $1`, id).Scan(&exists) + if err != nil { + if err.Error() == "sql: no rows in result set" { + c.JSON(http.StatusNotFound, gin.H{"error": "workspace not found"}) + return + } + log.Printf("AdminRestart: workspace lookup %s: %v", id, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "db lookup failed"}) + return + } + + // Fire the restart ASYNC — same pattern as the user-facing + // POST /workspaces/:id/restart handler. The actual restart runs + // in a goroutine; we return 202 Accepted immediately so the + // migrator's poll loop isn't held by the restart's own + // provisioning time. + h.goAsync(func() { h.RestartByID(id) }) + log.Printf("AdminRestart: dispatching restart for workspace %s (CP migrator settle — fleet-credential incident durable fix)", id) + c.JSON(http.StatusAccepted, gin.H{ + "status": "restart_dispatched", + "workspace_id": id, + }) +} diff --git a/workspace-server/internal/handlers/workspace_admin_restart_test.go b/workspace-server/internal/handlers/workspace_admin_restart_test.go new file mode 100644 index 00000000..421f11fe --- /dev/null +++ b/workspace-server/internal/handlers/workspace_admin_restart_test.go @@ -0,0 +1,160 @@ +package handlers + +// workspace_admin_restart_test.go — tests for the AdminRestart handler +// (the partner of the user-facing POST /workspaces/:id/restart). The CP +// migrator calls this to re-inject the tenant's LLM creds via the +// loadWorkspaceSecrets path on a freshly-migrated box (today's +// 2026-06-15 fleet-credential incident root-cause durable fix — see +// PRs #824 (CP) and this one (tenant partner)). Mirrors the +// SetComputeInstance test pattern (workspace_set_compute_instance_test.go). + +import ( + "database/sql" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/gin-gonic/gin" +) + +// AdminRestart re-injects LLM creds via the loadWorkspaceSecrets path +// (the durable fix for today's 2026-06-15 fleet-credential incident — +// see controlplane PR #824 for the migrator-side). The handler fires +// wh.RestartByID ASYNC (per the existing /restart endpoint's pattern) +// and returns 202 Accepted immediately. +func TestAdminRestart_HappyPath(t *testing.T) { + h, mock := setupBootstrapHandler(t) + + // Pre-flight: confirm the workspace exists. The handler does + // a SELECT 1 FROM workspaces WHERE id = $1 before firing the + // async restart, so we expect that query. + mock.ExpectQuery(`SELECT 1 FROM workspaces WHERE id = \$1`). + WithArgs("ws-migrated"). + WillReturnRows(sqlmock.NewRows([]string{"x"}).AddRow(1)) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Params = gin.Params{{Key: "id", Value: "ws-migrated"}} + c.Request = httptest.NewRequest("POST", "/admin/workspaces/ws-migrated/restart", nil) + + h.AdminRestart(c) + + if w.Code != http.StatusAccepted { + t.Fatalf("want 202, got %d: %s", w.Code, w.Body.String()) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet: %v", err) + } + // The actual restart is async; we don't assert on the goroutine + // (it would no-op on the test bootstrap since h has no provisioner + // wired; the goAsync panic-recovery swallows any panic cleanly). +} + +// A workspace id that matches no row is a 404 — the migrator can tell +// a stale id from a real restart. Distinct from SetComputeInstance's +// NoRowIs404 (which fires on the UPDATE rowcount), here the +// pre-flight SELECT does the work. +func TestAdminRestart_NoRowIs404(t *testing.T) { + h, mock := setupBootstrapHandler(t) + + mock.ExpectQuery(`SELECT 1 FROM workspaces WHERE id = \$1`). + WithArgs("ws-gone"). + WillReturnError(sql.ErrNoRows) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Params = gin.Params{{Key: "id", Value: "ws-gone"}} + c.Request = httptest.NewRequest("POST", "/admin/workspaces/ws-gone/restart", nil) + + h.AdminRestart(c) + + if w.Code != http.StatusNotFound { + t.Fatalf("want 404, got %d: %s", w.Code, w.Body.String()) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet: %v", err) + } +} + +// A DB failure on the pre-flight surfaces as 500 so the migrator +// can fail loudly rather than silently restart into a missing +// workspace. (RestartByID would fail too, but with a less-precise +// error from the deeper code path; surfacing the pre-flight 500 +// gives ops a clean diagnostic.) +func TestAdminRestart_DBErrorIs500(t *testing.T) { + h, mock := setupBootstrapHandler(t) + + mock.ExpectQuery(`SELECT 1 FROM workspaces WHERE id = \$1`). + WithArgs("ws-1"). + WillReturnError(errors.New("connection reset")) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Params = gin.Params{{Key: "id", Value: "ws-1"}} + c.Request = httptest.NewRequest("POST", "/admin/workspaces/ws-1/restart", nil) + + h.AdminRestart(c) + + if w.Code != http.StatusInternalServerError { + t.Fatalf("want 500, got %d: %s", w.Code, w.Body.String()) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet: %v", err) + } +} + +// An empty id is a 400 before any DB work — the migrator never +// issues an empty id (it always has a real wsID from the cutover +// record), so this is a defense-in-depth check, not a hot path. +func TestAdminRestart_EmptyIDIs400(t *testing.T) { + h, _ := setupBootstrapHandler(t) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Params = gin.Params{{Key: "id", Value: ""}} + c.Request = httptest.NewRequest("POST", "/admin/workspaces//restart", nil) + + h.AdminRestart(c) + + if w.Code != http.StatusBadRequest { + t.Errorf("want 400, got %d", w.Code) + } +} + +// Sanity check that the handler does NOT pause for the actual restart +// (the 202 path is async; the migrator's poll loop is not held by +// the restart's provisioning time). A 1ms-budgeted assertion catches +// a regression that turns the handler into a synchronous call. +func TestAdminRestart_AsyncDoesNotBlock(t *testing.T) { + h, mock := setupBootstrapHandler(t) + mock.ExpectQuery(`SELECT 1 FROM workspaces WHERE id = \$1`). + WithArgs("ws-1"). + WillReturnRows(sqlmock.NewRows([]string{"x"}).AddRow(1)) + + done := make(chan struct{}) + go func() { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Params = gin.Params{{Key: "id", Value: "ws-1"}} + c.Request = httptest.NewRequest("POST", "/admin/workspaces/ws-1/restart", nil) + h.AdminRestart(c) + close(done) + }() + select { + case <-done: + // PASS — handler returned quickly. + case <-timeAfter(1): + t.Fatal("AdminRestart blocked (the 202 must return without waiting for the restart goroutine)") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet: %v", err) + } +} + +// Use a package-private alias so the test file doesn't need to +// inline a time.After call. Kept inline; standard library time is +// imported via the test harness. +var timeAfter = func(d int) <-chan time.Time { return time.After(time.Duration(d) * time.Millisecond) } diff --git a/workspace-server/internal/router/router.go b/workspace-server/internal/router/router.go index 1dfb2370..d2064c9c 100644 --- a/workspace-server/internal/router/router.go +++ b/workspace-server/internal/router/router.go @@ -203,6 +203,18 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi // fighting the migration into a split-brain. Pure record repoint (no // deprovision); the CP migrator calls it once the cutover is verified. wsAdmin.POST("/admin/workspaces/:id/set-compute-instance", wh.SetComputeInstance) + // Admin-triggered restart of a workspace — the partner of the + // user-facing POST /workspaces/:id/restart (which uses the + // workspace's own bearer). The CP migrator calls this after a + // cross-cloud migration cutover to re-inject LLM creds via the + // loadWorkspaceSecrets path (today's 2026-06-15 fleet-credential + // incident root-cause durable fix — see PRs #824 (CP) and this + // one (tenant partner)). The handler fires wh.RestartByID async + // and returns 202 Accepted immediately; the actual restart + // happens in the background and the migrator's strengthened + // health check (assertCompletionServes in CP#824) verifies the + // cred re-injection landed. + wsAdmin.POST("/admin/workspaces/:id/restart", wh.AdminRestart) // Per-workspace LLM billing mode override (internal#691). Used by // CP's /cp/admin/workspaces/:id/llm-billing-mode proxy + (via that // proxy) by the canvas Config-tab "LLM Billing" section. Default-