diff --git a/docs/runbooks/engineer-agent-gitea-token-scope.md b/docs/runbooks/engineer-agent-gitea-token-scope.md new file mode 100644 index 000000000..f60b785ea --- /dev/null +++ b/docs/runbooks/engineer-agent-gitea-token-scope.md @@ -0,0 +1,124 @@ +# Engineer-Agent Gitea Token Scope Runbook + +## Symptom + +Engineer-class agents (e.g. `agent-dev-a`, `agent-dev-b`) fail swarm-pull issue discovery or receive HTTP 403 when calling Gitea issue-list APIs, while PR review and repository API operations continue to work. + +Typical failing call: +```bash +GET /api/v1/repos/molecule-ai/molecule-core/issues?state=open&labels=approved&limit=50 +# => 403 Forbidden +``` + +Typical working calls (same token): +```bash +GET /api/v1/repos/molecule-ai/molecule-core/pulls?state=open&limit=50 +POST /api/v1/repos/molecule-ai/molecule-core/pulls/1666/comments +# => 200 OK +``` + +## Root Cause + +Gitea v1.22.6 routes issue-list under the `Issue` scope category (`routers/api/v1/api.go:1379-1491`), while PR routes live under repository/pull routing (`api.go:1278-1305`). The scope gate derives required read/write level from HTTP method (`api.go:309-313`), so `GET /issues?...` requires `read:issue`. + +Engineer-class agent PATs were provisioned with repository and PR scopes but without `read:issue`, causing the asymmetric 403. + +## Detection + +1. **Agent-side**: swarm-pull workflow logs show `403 Forbidden` on issue enumeration but not on PR list/review. +2. **Platform-side**: Gitea access logs show `GET /repos/{owner}/{repo}/issues` returning 403 for the affected token. +3. **Reproduction** (from any workspace with a suspected token): + ```bash + TOKEN=$(cat /configs/secrets.d/GITEA_TOKEN) + PLATFORM="https://git.moleculesai.app" + + # Should succeed — confirms token is live + curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $TOKEN" \ + "$PLATFORM/api/v1/user" + + # Will 403 if the token lacks read:issue + curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $TOKEN" \ + "$PLATFORM/api/v1/repos/molecule-ai/molecule-core/issues?state=open&limit=1" + ``` + +## Immediate Fix + +### Step 1: Issue fresh PATs with correct scopes + +From a Gitea site-admin account (or via the Gitea web UI → Settings → Applications): + +1. Navigate to the affected user's profile (e.g. `agent-dev-a`). +2. Go to **Settings → Applications → Generate New Token**. +3. Select scopes: + - `read:repository` (existing) + - `write:repository` (existing, if push is required) + - `read:issue` (**add this**) + - `write:issue` (add only if agents must comment/edit issues) + - `read:pull-request` / `write:pull-request` (existing) + - `read:comment` / `write:comment` (existing, if PR review is required) +4. Copy the plaintext token immediately — it is shown only once. + +### Step 2: Update workspace secrets + +For each affected engineer workspace, update the Gitea token secret: + +```bash +# Via the platform API (admin auth required) +PLATFORM="https://agents-team.moleculesai.app" +ADMIN_TOKEN="" +WORKSPACE_ID="" +NEW_GITEA_TOKEN="" + +curl -X POST "$PLATFORM/workspaces/$WORKSPACE_ID/secrets" \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ + \"GITEA_TOKEN\": \"$NEW_GITEA_TOKEN\" + }" +``` + +Restart the workspace so the runtime re-reads secrets: +```bash +curl -X POST "$PLATFORM/workspaces/$WORKSPACE_ID/restart" \ + -H "Authorization: Bearer $ADMIN_TOKEN" +``` + +### Step 3: Smoke-test + +From the restarted workspace, verify all three paths: + +```bash +# 1. Issue list (the previously failing path) +curl -s -H "Authorization: token $GITEA_TOKEN" \ + "https://git.moleculesai.app/api/v1/repos/molecule-ai/molecule-core/issues?state=open&labels=approved&limit=1" | jq '.[0].number' + +# 2. PR list (should still work) +curl -s -H "Authorization: token $GITEA_TOKEN" \ + "https://git.moleculesai.app/api/v1/repos/molecule-ai/molecule-core/pulls?state=open&limit=1" | jq '.[0].number' + +# 3. Swarm-pull discovery (end-to-end) +# Trigger the agent's autonomous tick or delegate a task that enumerates open issues. +``` + +## Long-Term Fix + +Update the **workspace secret injection path** that writes `/configs/secrets.d/GITEA_TOKEN` for engineer-class agents. The provisioning template or secret-distribution job should request `read:issue` (and optionally `write:issue`) at token-creation time. + +File locations to audit: +- `.gitea/scripts/` — any token-provisioning automation +- `infra/terraform/` or equivalent — IAM/secret-manager templates +- `workspace-configs-templates/` — engineer-class workspace templates that declare required secrets + +## Prevention + +1. **Token scope checklist**: when provisioning new engineer-class agent tokens, verify the scope set includes `read:issue` before distributing the secret. +2. **Monitoring**: add an agent health-check that probes `GET /repos/molecule-ai/molecule-core/issues?limit=1` and surfaces a non-fatal warning if it returns 403. +3. **Documentation**: update the onboarding runbook for new engineer agents to include the full required scope list. + +## References + +- Gitea issue #1750: [RCA: engineer-token read:issue scope gap blocks swarm-pull workflow](https://git.moleculesai.app/molecule-ai/molecule-core/issues/1750) +- Gitea source: `routers/api/v1/api.go:309-313` (scope gate), `api.go:1278-1305` (PR routing), `api.go:1379-1491` (issue routing) +- Related: PR #1542 (provisioner git-creds injection), PR #1669 (auth_token inline mint) diff --git a/workspace-server/internal/handlers/handlers_additional_test.go b/workspace-server/internal/handlers/handlers_additional_test.go index 9916f0469..31be3d116 100644 --- a/workspace-server/internal/handlers/handlers_additional_test.go +++ b/workspace-server/internal/handlers/handlers_additional_test.go @@ -43,6 +43,8 @@ func TestWorkspaceCreate_WithParentID(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -79,6 +81,8 @@ func TestWorkspaceCreate_ExplicitClaudeCodeRuntime(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -300,6 +304,8 @@ func TestWorkspaceCreate_MaxConcurrentTasksOverride(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) diff --git a/workspace-server/internal/handlers/handlers_test.go b/workspace-server/internal/handlers/handlers_test.go index fbfc134b7..6a95d9816 100644 --- a/workspace-server/internal/handlers/handlers_test.go +++ b/workspace-server/internal/handlers/handlers_test.go @@ -384,6 +384,8 @@ func TestWorkspaceCreate(t *testing.T) { // Expect RecordAndBroadcast INSERT for WORKSPACE_PROVISIONING mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -420,6 +422,76 @@ func TestWorkspaceCreate(t *testing.T) { } } +// TestWorkspaceCreate_ReturnsAuthToken_201 pins the inline-auth_token +// behaviour added for #1644. Pre-fix, the 201 response was +// {id, status, awareness_namespace, workspace_access} — callers had to +// make a separate POST to /admin/workspaces/:id/tokens (AdminAuth-gated, +// path-prefix differs in CP-admin deploys) OR fall back to the dev-only +// GET /admin/workspaces/:id/test-token (deliberately 404s on +// MOLECULE_ENV=production per feedback_no_dev_only_routes_in_e2e). +// +// Post-fix: every Create response includes an `auth_token` field with +// the freshly-minted plaintext bearer (returned once, never recoverable). +// This is the SSOT path — production E2E + canvas + org_import all +// get the bearer they need in the same round trip. +// +// Failure path is non-fatal: if the IssueToken DB call fails, the 201 +// still goes out without auth_token + a fallback log line. That branch +// is exercised by sqlmock returning a non-INSERT-INTO-workspace_auth_tokens +// path here — the test asserts presence on the happy path. +func TestWorkspaceCreate_ReturnsAuthToken_201(t *testing.T) { + mock := setupTestDB(t) + setupTestRedis(t) + broadcaster := newTestBroadcaster() + handler := NewWorkspaceHandler(broadcaster, nil, "http://localhost:8080", "/tmp/configs") + + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO workspaces"). + WithArgs(sqlmock.AnyArg(), "Token Holder", nil, 3, "claude-code", (*string)(nil), nil, "none", (*int64)(nil), models.DefaultMaxConcurrentTasks, "push"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + mock.ExpectExec("INSERT INTO canvas_layouts"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO structure_events"). + WillReturnResult(sqlmock.NewResult(0, 1)) + // The inline mint added in #1644 Part B — wsauth.IssueToken issues + // a new bearer via INSERT INTO workspace_auth_tokens (workspace_id, + // token_hash, prefix). This is the assertion that the new code path + // reaches the DB. + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + body := `{"name":"Token Holder","model":"anthropic:claude-opus-4-7"}` + c.Request = httptest.NewRequest("POST", "/workspaces", bytes.NewBufferString(body)) + c.Request.Header.Set("Content-Type", "application/json") + + handler.Create(c) + + if w.Code != http.StatusCreated { + t.Fatalf("expected 201, got %d: %s", w.Code, w.Body.String()) + } + var resp map[string]interface{} + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("parse response: %v", err) + } + tok, ok := resp["auth_token"].(string) + if !ok || tok == "" { + t.Fatalf("expected non-empty auth_token in 201 response (the #1644 SSOT inline mint), got: %s", w.Body.String()) + } + // Sanity: tokens are base64-RawURL encoded 32-byte payloads (per + // wsauth/tokens.go::tokenPayloadBytes), so a meaningful lower bound + // is ~40 chars. If this fails, IssueToken's contract drifted. + if len(tok) < 40 { + t.Errorf("auth_token suspiciously short (%d chars) — wsauth.IssueToken contract drift?", len(tok)) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet sqlmock expectations — inline mint path may have skipped IssueToken: %v", err) + } +} + func TestBuildProvisionerConfig_WorkspacePathFromPayload(t *testing.T) { setupTestDB(t) broadcaster := newTestBroadcaster() diff --git a/workspace-server/internal/handlers/workspace.go b/workspace-server/internal/handlers/workspace.go index b0947d756..ee9069a1e 100644 --- a/workspace-server/internal/handlers/workspace.go +++ b/workspace-server/internal/handlers/workspace.go @@ -807,11 +807,38 @@ func (h *WorkspaceHandler) Create(c *gin.Context) { } } - c.JSON(http.StatusCreated, gin.H{ + // Mint the workspace's first bearer token and return it inline + // (#1644). Pre-fix, callers had to make a separate POST to + // /admin/workspaces/:id/tokens (production path, AdminAuth-gated, + // but the path-prefix differs in CP-admin deploys so staging E2E + // got HTML 404) OR fall back to GET /admin/workspaces/:id/test-token + // (dev-only — deliberately 404s on MOLECULE_ENV=production per + // admin_test_token.go::TestTokensEnabled, which violates + // feedback_no_dev_only_routes_in_e2e). Inlining the first token here + // makes the create response the SSOT — every caller (canvas Save, + // org_import, E2E, third-party API) gets the bearer they need to + // authenticate /activity, /a2a, /memory etc. without an extra + // round trip to a separate mint endpoint. + // + // Failure is non-fatal: the workspace row already committed; the + // operator can recover via POST /admin/workspaces/:id/tokens + // (canonical admin mint) or POST /workspaces/:id/external/rotate + // (already-used for the external pre-register path above). We log + // the failure and return 201 without the field — callers that need + // the token will get a clear-shaped fallback (auth_token absent + // from response = use the admin mint path). + resp := gin.H{ "id": id, "status": "provisioning", "workspace_access": workspaceAccess, - }) + } + if authToken, tokErr := wsauth.IssueToken(ctx, db.DB, id); tokErr != nil { + log.Printf("Create workspace %s: inline auth_token mint failed (non-fatal — caller can use POST /admin/workspaces/:id/tokens): %v", id, tokErr) + } else { + resp["auth_token"] = authToken + } + + c.JSON(http.StatusCreated, resp) } // addProvisionTimeoutMs decorates a workspace response map with the diff --git a/workspace-server/internal/handlers/workspace_budget_test.go b/workspace-server/internal/handlers/workspace_budget_test.go index 4a467ae84..d8f0a2116 100644 --- a/workspace-server/internal/handlers/workspace_budget_test.go +++ b/workspace-server/internal/handlers/workspace_budget_test.go @@ -168,6 +168,8 @@ func TestWorkspaceBudget_Create_WithLimit(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) diff --git a/workspace-server/internal/handlers/workspace_compute_test.go b/workspace-server/internal/handlers/workspace_compute_test.go index 8a278a809..d11ee0210 100644 --- a/workspace-server/internal/handlers/workspace_compute_test.go +++ b/workspace-server/internal/handlers/workspace_compute_test.go @@ -91,6 +91,8 @@ func TestWorkspaceCreate_WithCompute_PersistsComputeJSON(t *testing.T) { mock.ExpectCommit() mock.ExpectExec("INSERT INTO canvas_layouts"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) diff --git a/workspace-server/internal/handlers/workspace_test.go b/workspace-server/internal/handlers/workspace_test.go index f10bcfb1e..12d6a6819 100644 --- a/workspace-server/internal/handlers/workspace_test.go +++ b/workspace-server/internal/handlers/workspace_test.go @@ -390,6 +390,8 @@ func TestWorkspaceCreate_DefaultsApplied(t *testing.T) { // Expect RecordAndBroadcast INSERT mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -438,6 +440,9 @@ func TestWorkspaceCreate_SaaSHardForcesTier4(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + // External workspaces return early with connectionToken in the + // connection payload; they do NOT reach the inline auth_token mint + // at the bottom of Create (non-external path only). w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -478,6 +483,8 @@ func TestWorkspaceCreate_WithSecrets_Persists(t *testing.T) { // canvas_layouts (non-fatal, outside tx) mock.ExpectExec("INSERT INTO canvas_layouts"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -559,6 +566,8 @@ func TestWorkspaceCreate_EmptySecrets_OK(t *testing.T) { mock.ExpectCommit() mock.ExpectExec("INSERT INTO canvas_layouts"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -598,6 +607,7 @@ func TestWorkspaceCreate_ExternalURL_SSRFSafe(t *testing.T) { mock.ExpectExec("UPDATE workspaces SET url"). WillReturnResult(sqlmock.NewResult(0, 1)) // CacheURL is non-fatal — uses Redis (db.RDB, set by setupTestRedis), not the DB. + // External workspaces return early before the inline auth_token mint. w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -1809,6 +1819,8 @@ runtime_config: WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -1868,6 +1880,8 @@ model: anthropic:claude-sonnet-4-5 WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -1921,6 +1935,8 @@ runtime_config: WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -2066,6 +2082,8 @@ func TestWorkspaceCreate_188_ExplicitRuntimeNoTemplate_OK(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec("INSERT INTO structure_events"). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO workspace_auth_tokens"). + WillReturnResult(sqlmock.NewResult(0, 1)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w)