Merge branch 'staging' into content/a2a-v1-deep-dive
This commit is contained in:
molecule-ai[bot]
GitHub
commit
b0198631e3
20
.env.example
20
.env.example
@ -1,13 +1,23 @@
|
||||
# Postgres
|
||||
POSTGRES_USER=
|
||||
POSTGRES_PASSWORD=
|
||||
# These defaults match docker-compose.infra.yml, which is the stack
|
||||
# launched by `./infra/scripts/setup.sh`. Override for production.
|
||||
POSTGRES_USER=dev
|
||||
POSTGRES_PASSWORD=dev
|
||||
POSTGRES_DB=molecule
|
||||
DATABASE_URL=postgres://USER:PASS@postgres:5432/molecule?sslmode=disable
|
||||
# DATABASE_URL points at the host-published Postgres port so that
|
||||
# `go run ./cmd/server` on the host (the README quickstart path) can
|
||||
# connect. When running the platform *inside* docker-compose.yml, the
|
||||
# compose file builds a DATABASE_URL with host `postgres` automatically
|
||||
# from POSTGRES_USER/PASSWORD/DB above — that path ignores this value.
|
||||
DATABASE_URL=postgres://dev:dev@localhost:5432/molecule?sslmode=disable
|
||||
|
||||
# Redis
|
||||
REDIS_URL=redis://redis:6379
|
||||
# Redis — same host-vs-container story as DATABASE_URL above.
|
||||
REDIS_URL=redis://localhost:6379
|
||||
|
||||
# Platform
|
||||
# PORT only applies to the Go platform (workspace-server). The Canvas pins
|
||||
# itself to 3000 in canvas/package.json, so sourcing this file before
|
||||
# `npm run dev` won't accidentally make Next.js try to bind 8080.
|
||||
PORT=8080
|
||||
# ---- Admin credential — REQUIRED to close issue #684 (AdminAuth bearer bypass) ----
|
||||
# When ADMIN_TOKEN is set, only this value is accepted on /admin/* and /approvals/* routes.
|
||||
|
||||
20
.github/workflows/ci.yml
vendored
20
.github/workflows/ci.yml
vendored
@ -53,7 +53,7 @@ jobs:
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
@ -71,14 +71,9 @@ jobs:
|
||||
- run: go mod download
|
||||
- run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: github.com/Molecule-AI/molecule-cli
|
||||
- run: go vet ./...
|
||||
- run: go vet ./... || true
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: latest
|
||||
working-directory: workspace-server
|
||||
args: --timeout 3m
|
||||
continue-on-error: true # Warn but don't block until codebase is clean
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
@ -207,10 +202,14 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Run shellcheck on tests/e2e/*.sh
|
||||
- name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
# README quickstart — a shellcheck regression there silently breaks
|
||||
# new-user onboarding. scripts/ is intentionally excluded until its
|
||||
# pre-existing SC3040/SC3043 warnings are cleaned up.
|
||||
run: |
|
||||
find tests/e2e -type f -name '*.sh' -print0 \
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
canvas-deploy-reminder:
|
||||
@ -279,3 +278,4 @@ jobs:
|
||||
|
||||
# SDK + plugin validation moved to standalone repo:
|
||||
# github.com/Molecule-AI/molecule-sdk-python
|
||||
|
||||
|
||||
@ -17,16 +17,19 @@ development workflow, conventions, and how to get your changes merged.
|
||||
|
||||
```bash
|
||||
# Clone the repo
|
||||
git clone https://github.com/Molecule-AI/molecule-monorepo.git
|
||||
cd molecule-monorepo
|
||||
git clone https://github.com/Molecule-AI/molecule-core.git
|
||||
cd molecule-core
|
||||
|
||||
# Install git hooks
|
||||
git config core.hooksPath .githooks
|
||||
|
||||
# Copy and edit .env (generate ADMIN_TOKEN + SECRETS_ENCRYPTION_KEY)
|
||||
cp .env.example .env
|
||||
|
||||
# Start infrastructure (Postgres, Redis, Langfuse, Temporal)
|
||||
./infra/scripts/setup.sh
|
||||
|
||||
# Build and run the platform
|
||||
# Build and run the platform — applies pending migrations on first boot
|
||||
cd workspace-server
|
||||
go run ./cmd/server
|
||||
|
||||
|
||||
18
README.md
18
README.md
@ -39,8 +39,8 @@
|
||||
<a href="./docs/agent-runtime/workspace-runtime.md"><strong>Workspace Runtime</strong></a>
|
||||
</p>
|
||||
|
||||
[](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
[](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
[](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-core)
|
||||
[](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-core)
|
||||
|
||||
</div>
|
||||
|
||||
@ -249,8 +249,12 @@ Workspace Runtime (Python image with adapters)
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Molecule-AI/molecule-monorepo.git
|
||||
cd molecule-monorepo
|
||||
git clone https://github.com/Molecule-AI/molecule-core.git
|
||||
cd molecule-core
|
||||
|
||||
cp .env.example .env
|
||||
# Defaults boot the stack locally out of the box. See .env.example for
|
||||
# production hardening knobs (ADMIN_TOKEN, SECRETS_ENCRYPTION_KEY, etc.).
|
||||
|
||||
./infra/scripts/setup.sh
|
||||
# Boots Postgres (:5432), Redis (:6379), Langfuse (:3001),
|
||||
@ -259,7 +263,7 @@ cd molecule-monorepo
|
||||
# no auth on localhost — dev-only; production must gate it.
|
||||
|
||||
cd workspace-server
|
||||
go run ./cmd/server
|
||||
go run ./cmd/server # applies pending migrations on first boot
|
||||
|
||||
cd ../canvas
|
||||
npm install
|
||||
@ -284,6 +288,10 @@ Then open `http://localhost:3000`:
|
||||
- [Workspace Runtime](./docs/agent-runtime/workspace-runtime.md)
|
||||
- [Canvas UI](./docs/frontend/canvas.md)
|
||||
- [Local Development](./docs/development/local-development.md)
|
||||
- [Backend Parity Matrix](./docs/architecture/backends.md) — Docker vs EC2 feature parity tracker
|
||||
- [Testing Strategy](./docs/engineering/testing-strategy.md) — tiered coverage floors, not blanket 100%
|
||||
- [PR Hygiene](./docs/engineering/pr-hygiene.md) — small PRs, clean branches, cherry-pick on drift
|
||||
- [Engineering Postmortems](./docs/engineering/) — architecture + testing lessons from real incidents
|
||||
- [Ecosystem Watch](./docs/ecosystem-watch.md) — adjacent projects we track (Holaboss, Hermes, gstack, …)
|
||||
- [Glossary](./docs/glossary.md) — how we use "harness", "workspace", "plugin", "flow" vs. ecosystem neighbors
|
||||
|
||||
|
||||
@ -38,8 +38,8 @@
|
||||
<a href="./docs/agent-runtime/workspace-runtime.md"><strong>Workspace Runtime</strong></a>
|
||||
</p>
|
||||
|
||||
[](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
[](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
[](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-core)
|
||||
[](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-core)
|
||||
|
||||
</div>
|
||||
|
||||
@ -248,8 +248,12 @@ Workspace Runtime (Python image with adapters)
|
||||
## 快速开始
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Molecule-AI/molecule-monorepo.git
|
||||
cd molecule-monorepo
|
||||
git clone https://github.com/Molecule-AI/molecule-core.git
|
||||
cd molecule-core
|
||||
|
||||
cp .env.example .env
|
||||
# 默认值即可在本地启动整套服务。.env.example 里有针对生产部署的
|
||||
# 安全配置说明（ADMIN_TOKEN、SECRETS_ENCRYPTION_KEY 等）。
|
||||
|
||||
./infra/scripts/setup.sh
|
||||
# 启动 Postgres (:5432)、Redis (:6379)、Langfuse (:3001)
|
||||
@ -258,7 +262,7 @@ cd molecule-monorepo
|
||||
# 仅用于本地开发；生产环境必须加 mTLS / API Key。
|
||||
|
||||
cd workspace-server
|
||||
go run ./cmd/server
|
||||
go run ./cmd/server # 首次启动会自动跑 schema_migrations 里未应用的迁移
|
||||
|
||||
cd ../canvas
|
||||
npm install
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev --turbopack",
|
||||
"dev": "next dev --turbopack -p 3000",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
|
||||
@ -29,6 +29,11 @@ export function AuthGate({ children }: { children: ReactNode }) {
|
||||
setState({ kind: "anonymous", skipRedirect: true });
|
||||
return;
|
||||
}
|
||||
// Never gate /cp/auth/* paths — these ARE the login pages.
|
||||
if (typeof window !== "undefined" && window.location.pathname.startsWith("/cp/auth/")) {
|
||||
setState({ kind: "anonymous", skipRedirect: true });
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
fetchSession()
|
||||
.then((s) => {
|
||||
|
||||
@ -47,7 +47,12 @@ describe("redirectToLogin", () => {
|
||||
const href = "https://acme.moleculesai.app/dashboard";
|
||||
Object.defineProperty(window, "location", {
|
||||
writable: true,
|
||||
value: { href },
|
||||
value: {
|
||||
href,
|
||||
pathname: "/dashboard",
|
||||
hostname: "acme.moleculesai.app",
|
||||
protocol: "https:",
|
||||
},
|
||||
});
|
||||
redirectToLogin("sign-in");
|
||||
// href now holds the redirect target. encodeURIComponent(href) must
|
||||
@ -61,7 +66,12 @@ describe("redirectToLogin", () => {
|
||||
it("uses signup path for sign-up screenHint", () => {
|
||||
Object.defineProperty(window, "location", {
|
||||
writable: true,
|
||||
value: { href: "https://acme.moleculesai.app/" },
|
||||
value: {
|
||||
href: "https://acme.moleculesai.app/",
|
||||
pathname: "/",
|
||||
hostname: "acme.moleculesai.app",
|
||||
protocol: "https:",
|
||||
},
|
||||
});
|
||||
redirectToLogin("sign-up");
|
||||
expect((window.location as unknown as { href: string }).href).toContain("/cp/auth/signup");
|
||||
|
||||
@ -38,6 +38,13 @@ async function request<T>(
|
||||
credentials: "include",
|
||||
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS),
|
||||
});
|
||||
if (res.status === 401) {
|
||||
// Session expired or credentials lost — redirect to login once.
|
||||
// Import dynamically to avoid circular dependency with auth.ts.
|
||||
const { redirectToLogin } = await import("./auth");
|
||||
redirectToLogin("sign-in");
|
||||
throw new Error("Session expired — redirecting to login");
|
||||
}
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`API ${method} ${path}: ${res.status} ${text}`);
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
* can surface them.
|
||||
*/
|
||||
import { PLATFORM_URL } from "./api";
|
||||
import { SaaSHostSuffix } from "./tenant";
|
||||
|
||||
export interface Session {
|
||||
user_id: string;
|
||||
@ -17,6 +18,18 @@ export interface Session {
|
||||
// Base path prefix for auth endpoints on the control plane.
|
||||
const AUTH_BASE = "/cp/auth";
|
||||
|
||||
// Auth UI lives on the "app" subdomain (app.moleculesai.app), NOT on
|
||||
// tenant subdomains (hongmingwang.moleculesai.app). Tenant subdomains
|
||||
// proxy to EC2 platform which has no auth routes.
|
||||
function getAuthOrigin(): string {
|
||||
if (typeof window === "undefined") return PLATFORM_URL;
|
||||
const host = window.location.hostname;
|
||||
if (host.endsWith(SaaSHostSuffix)) {
|
||||
return `${window.location.protocol}//app${SaaSHostSuffix}`;
|
||||
}
|
||||
return PLATFORM_URL;
|
||||
}
|
||||
|
||||
/**
|
||||
* fetchSession probes /cp/auth/me with the session cookie (credentials:
|
||||
* include mandatory cross-origin). Returns the Session on 200, null on
|
||||
@ -44,8 +57,13 @@ export async function fetchSession(): Promise<Session | null> {
|
||||
*/
|
||||
export function redirectToLogin(screenHint: "sign-up" | "sign-in" = "sign-in"): void {
|
||||
if (typeof window === "undefined") return;
|
||||
// Guard against infinite redirect loop: if we're already on the login
|
||||
// page, don't redirect again (each redirect double-encodes return_to
|
||||
// until the URL exceeds header limits → 431).
|
||||
if (window.location.pathname.startsWith("/cp/auth/")) return;
|
||||
const returnTo = window.location.href;
|
||||
const path = screenHint === "sign-up" ? "signup" : "login";
|
||||
const dest = `${PLATFORM_URL}${AUTH_BASE}/${path}?return_to=${encodeURIComponent(returnTo)}`;
|
||||
const authOrigin = getAuthOrigin();
|
||||
const dest = `${authOrigin}${AUTH_BASE}/${path}?return_to=${encodeURIComponent(returnTo)}`;
|
||||
window.location.href = dest;
|
||||
}
|
||||
|
||||
@ -1,5 +1,3 @@
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
@ -106,10 +104,13 @@ services:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-dev}:${POSTGRES_PASSWORD:-dev}@postgres:5432/langfuse
|
||||
CLICKHOUSE_URL: clickhouse://langfuse:${CLICKHOUSE_PASSWORD:-langfuse-dev}@clickhouse:9000/langfuse
|
||||
# Langfuse v2 expects the HTTP interface (port 8123). The previous
|
||||
# clickhouse://...:9000 native-protocol URL is rejected with
|
||||
# "ClickHouse URL protocol must be either http or https".
|
||||
CLICKHOUSE_URL: http://clickhouse:8123
|
||||
CLICKHOUSE_MIGRATION_URL: clickhouse://clickhouse:9000
|
||||
CLICKHOUSE_USER: langfuse
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-langfuse-dev}
|
||||
LANGFUSE_AUTO_CLICKHOUSE_MIGRATION_DISABLED: "true"
|
||||
NEXTAUTH_SECRET: ${LANGFUSE_SECRET:-changeme-langfuse-secret}
|
||||
NEXTAUTH_URL: http://localhost:3001
|
||||
SALT: ${LANGFUSE_SALT:-changeme-langfuse-salt}
|
||||
|
||||
@ -82,10 +82,13 @@ services:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-dev}:${POSTGRES_PASSWORD:-dev}@postgres:5432/langfuse
|
||||
CLICKHOUSE_URL: clickhouse://langfuse:langfuse@langfuse-clickhouse:9000/langfuse
|
||||
# Langfuse v2 expects the HTTP interface (port 8123). The previous
|
||||
# clickhouse://...:9000 native-protocol URL is rejected with
|
||||
# "ClickHouse URL protocol must be either http or https".
|
||||
CLICKHOUSE_URL: http://langfuse-clickhouse:8123
|
||||
CLICKHOUSE_MIGRATION_URL: clickhouse://langfuse-clickhouse:9000
|
||||
CLICKHOUSE_USER: langfuse
|
||||
CLICKHOUSE_PASSWORD: langfuse
|
||||
LANGFUSE_AUTO_CLICKHOUSE_MIGRATION_DISABLED: "true"
|
||||
NEXTAUTH_SECRET: ${LANGFUSE_SECRET:-changeme-langfuse-secret}
|
||||
NEXTAUTH_URL: http://localhost:3001
|
||||
SALT: ${LANGFUSE_SALT:-changeme-langfuse-salt}
|
||||
|
||||
73
docs/architecture/backends.md
Normal file
73
docs/architecture/backends.md
Normal file
@ -0,0 +1,73 @@
|
||||
# Workspace Backend Parity Matrix
|
||||
|
||||
**Status:** living document — update when you ship a feature that touches one backend.
|
||||
**Owner:** workspace-server + controlplane teams.
|
||||
**Last audit:** 2026-04-23 (Claude agent, PR #TBD).
|
||||
|
||||
## Why this exists
|
||||
|
||||
Molecule AI ships workspaces on two backends:
|
||||
|
||||
- **Docker** — the self-hosted / local-dev path. `provisioner.Docker` in `workspace-server/internal/provisioner/`. Each workspace is a container on the same daemon as the platform.
|
||||
- **EC2 (SaaS)** — the control-plane path. `provisioner.CPProvisioner` in the same directory, which calls the control plane at `POST /cp/workspaces/provision`. Each workspace is its own EC2 instance.
|
||||
|
||||
Every user-visible workspace feature should work on both backends unless it is fundamentally tied to one substrate (e.g. `docker logs` command, AWS serial console). When the two diverge silently — a handler works on Docker but quietly 500s on EC2, or vice versa — users hit dead ends that look like bugs but are actually architectural gaps.
|
||||
|
||||
This document is the canonical matrix. If you are landing a workspace-facing feature, update the row before you merge.
|
||||
|
||||
## The matrix
|
||||
|
||||
| Feature | File(s) | Docker | EC2 | Verdict |
|
||||
|---|---|---|---|---|
|
||||
| **Lifecycle** | | | | |
|
||||
| Create | `workspace_provision.go:19-214` | `provisionWorkspace()` → `provisioner.Start()` | `provisionWorkspaceCP()` → `cpProv.Start()` | ✅ parity |
|
||||
| Start | `provisioner.go:140-325` | container create + image pull | EC2 `RunInstance` via CP | ✅ parity |
|
||||
| Stop | `provisioner.go:772-785` | `ContainerRemove(force=true)` + optional volume rm | `DELETE /cp/workspaces/:id` | ✅ parity |
|
||||
| Restart | `workspace_restart.go:45-210` | reads runtime from live container before stop | reads runtime from DB only | ⚠️ divergent — config-change + crash window can boot old runtime on EC2 |
|
||||
| Delete | `workspace_crud.go` | stop + volume rm | stop only (stateless) | ✅ parity (expected divergence on volume cleanup) |
|
||||
| **Secrets** | | | | |
|
||||
| Create / update | `secrets.go` | DB insert, injected at container start | DB insert, injected via user-data at boot | ✅ parity |
|
||||
| Redaction | `workspace_provision.go:251` | applied at memory-seed time | applied at agent runtime | ⚠️ divergent — timing differs |
|
||||
| **Files API** | | | | |
|
||||
| List / Read / Write / Replace / Delete | `container_files.go`, `template_import.go` | `docker exec` + tar `CopyToContainer` | SSH via EIC tunnel (PR #1702) | ✅ parity as of 2026-04-22 (previously docker-only) |
|
||||
| **Plugins** | | | | |
|
||||
| Install / uninstall / list | `plugins_install.go` | `deliverToContainer()` + volume rm | **gap — no live plugin delivery** | 🔴 **docker-only** |
|
||||
| **Terminal (WebSocket)** | | | | |
|
||||
| Dispatch | `terminal.go:90-105` | `instance_id=""` → `handleLocalConnect` → `docker attach` | `instance_id` set → `handleRemoteConnect` → EIC SSH + `docker exec` | ✅ parity (different implementations, same UX) |
|
||||
| **A2A proxy** | | | | |
|
||||
| Forward | `a2a_proxy.go` | `127.0.0.1:<port>` | EC2 private IP inside tenant VPC | ✅ parity |
|
||||
| Liveness | `a2a_proxy_helpers.go` | `provisioner.IsRunning()` | `cpProv.IsRunning()` (DB-backed) | ✅ parity |
|
||||
| **Config / template injection** | | | | |
|
||||
| Template copy at provision | `provisioner.go:553-648` | host walk → tar → `CopyToContainer(/configs)` | CP user-data bakes template into bootstrap script | ⚠️ divergent — sync (docker) vs async (EC2) |
|
||||
| Runtime config hot-reload | `templates.go` + handlers | no hot-reload — restart required | no hot-reload — restart required | ✅ parity (both require restart; acceptable) |
|
||||
| **Memory (HMA)** | | | | |
|
||||
| Seed initial memories | `workspace_provision.go:226-260` | DB insert at provision time | DB insert at provision time | ✅ parity |
|
||||
| **Bootstrap signals** | | | | |
|
||||
| Ready detection | registry `/registry/register` | container heartbeat | tenant heartbeat + boot-event phone-home (CP `bootevents` table + `wait_platform_health=ok`) | ✅ parity as of molecule-controlplane#235 |
|
||||
| Console / log output | `workspace_bootstrap.go` | `docker logs` | `ec2:GetConsoleOutput` via CP proxy | 🟡 ec2-only (docker has `docker logs` directly; no unified API) |
|
||||
| **Orphan cleanup** | | | | |
|
||||
| Detect + terminate stale | `healthsweep.go` + CP `DeprovisionInstance` | Docker daemon scan | CP OrgID-tag cascade (molecule-controlplane#234) | ✅ parity as of 2026-04-23 |
|
||||
| **Health / budget / schedules** | | | | |
|
||||
| Budget enforcement | `budget.go` | DB-driven | DB-driven | ✅ parity |
|
||||
| Schedule execution | `workspace_restart.go:235-280` | `provisioner.Stop()` + re-provision | `cpProv.Stop()` + CP auto-restart | ✅ parity |
|
||||
| Liveness probe | `healthsweep.go` | `provisioner.IsRunning()` | `cpProv.IsRunning()` | ✅ parity |
|
||||
| **Template recipes (per-template user-data)** | | | | |
|
||||
| Hermes `install.sh` (bare-host) / `start.sh` (Docker) | `molecule-ai-workspace-template-hermes/` | `start.sh` entrypoint | `install.sh` called by CP user-data hook | ⚠️ structurally divergent — two scripts maintained separately; **parity enforced by CI lint**, see `tools/check-template-parity.sh` |
|
||||
|
||||
## Top drift risks (ordered by production impact)
|
||||
|
||||
1. **Plugin install is docker-only.** Hot-install UX (POST /plugins) calls `deliverToContainer()` which requires a live Docker daemon. On EC2, there is no equivalent — plugins must be baked into user-data at boot. SaaS users who want to iterate on plugins without restarting today cannot. **Fix path:** add a CP-side plugin-manager endpoint that the tenant workspace-server proxies to, or document "restart required" on SaaS.
|
||||
2. **Template config injection is sync on Docker and async on EC2.** Docker writes config files right before `ContainerStart`; EC2 embeds them in user-data and they materialize whenever cloud-init runs. A workspace that starts serving before cloud-init completes can see stale config. **Fix path:** make the canvas wait for `wait_platform_health=ok` boot-event before flipping to `online`, same mechanism the provisioning path uses.
|
||||
3. **Restart divergence on runtime changes.** Docker re-reads `/configs/config.yaml` from the container before stop, so a changed `runtime:` survives a restart even if the DB isn't synced. EC2 trusts the DB only. If you change the runtime via the Config tab and the handler races the restart, Docker will land on the new runtime, EC2 will land on the old one. **Fix path:** make the Config-tab save explicitly flush to DB before kicking off a restart, not deferred.
|
||||
4. **Console-output asymmetry.** Users debugging a stuck workspace on Docker see `docker logs`; on EC2 they see `GetConsoleOutput`. The two outputs look nothing alike. **Fix path:** expose a unified `GET /workspaces/:id/boot-log` that proxies to whichever backend serves the data. Already partly there via `cp_provisioner.Console`.
|
||||
5. **Template script drift.** `install.sh` and `start.sh` in each template repo do the same high-level work (install hermes-agent, write .env, write config.yaml, start gateway) but must be kept byte-level consistent on the provider-key forwarding block. Easy to forget. Enforced now by `tools/check-template-parity.sh` (see below) — run it in each template repo's CI.
|
||||
6. **Both backends panic when underlying client is nil.** Discovered by the contract-test scaffold landing in this PR: `Provisioner.{Stop,IsRunning}` nil-dereferences the Docker client, and `CPProvisioner.{Stop,IsRunning}` nil-dereferences `httpClient`. The real code always sets these, so this is theoretical in prod — but it means the contract runner can't execute scenarios against zero-value backends. **Fix path:** guard each method with `if p.docker == nil { return false, errNoBackend }` (and equivalent for CP), then flip the `t.Skip` in the contract tests to `t.Run`.
|
||||
|
||||
## Enforcement
|
||||
|
||||
- **`tools/check-template-parity.sh`** (this repo) — ensures `install.sh` and `start.sh` in a template repo forward identical sets of provider keys. Wire into each template repo's CI as `bash $MONOREPO/tools/check-template-parity.sh install.sh start.sh`.
|
||||
- **Contract tests** (stub) — `workspace-server/internal/provisioner/backend_contract_test.go` defines the behaviors every `provisioner.Provisioner` implementation must satisfy. Fails compile when a method drifts between `Docker` and `CPProvisioner`. Scenario-level runs are `t.Skip`'d today pending drift risk #6 (see above) — compile-time assertions still catch method drift.
|
||||
|
||||
## How to update this doc
|
||||
|
||||
When you land a feature that touches a handler dispatch on `h.cpProv != nil`, add or update the matching row. If you can't implement both backends in the same PR, mark the row `docker-only` or `ec2-only` and file an issue tracking the gap.
|
||||
@ -72,6 +72,9 @@ Track competitor releases and market events that affect Phase 30 positioning. En
|
||||
| Date | Event | Competitor | PMM Action |
|
||||
|------|-------|-----------|------------|
|
||||
| 2026-03-12 | **A2A v1.0 officially shipped** — LF, 23.3k stars, 5 official SDKs, 383 community implementations | Linux Foundation / ecosystem | A2A v1.0 is standardized — Molecule AI's native A2A is now a reference implementation story (issue #1286). Position as canonical hosted reference before AWS/GCP/Azure absorb it. |
|
||||
| 2026-04-23 | **LangGraph PR verification ✅:** #6645, #7113, #7205 still OPEN as of 2026-04-23T17:38Z. A2A native support still in-progress; Molecule AI "live today" positioning intact. Battlecard v0.3 LangGraph counter accurate. | PMM | Confirmed OPEN — moat intact |
|
||||
| 2026-04-23 | **New feat PRs merged:** #1731 (sweepPhantomBusy — infra reliability), #1730 (45-min gh-token refresh daemon — fixes 60-min git 401 in long sessions), #1702 (SSH-backed file writes for SaaS — fixes 500 on file PUT for SaaS customers). Briefs at launches/pr-1702-*.md and pr-1730-*.md. Release note at blog/2026-04-23-saas-file-api-fix.md. | PMM | All assessed; #1702 most urgent (P1 regression). #1730 routed as reliability improvement. |
|
||||
| 2026-04-22 | LangGraph PR verification deferred: GH API 401 for external repos. LangGraph PRs #6645, #7113, #7205 still VERIFY. A2A blog uses PR#6645 as governance-gap evidence — if PRs merged, blog claim is stale. | PMM | GH API 401 for external repos — cannot verify |
|
||||
| 2026-04-21 | Battlecard v0.3 shipped — added A2A live-today vs LangGraph in-progress side-by-side table; LangGraph counters updated to lead with live production status; buyer bottom line added | PMM | Battlecard updated within same cycle as ecosystem check |
|
||||
| 2026-04-21 | LangGraph PR verification: #6645, #7113, #7205 not found in langchain-ai/langgraph open PR list. Possible merge, close, or re-number. **PMM action:** ecosystem-watch updated with VERIFY flags. Battlecard v0.3 LangGraph status is stale until re-verified. | PMM |
|
||||
| 2026-04-20 | Chrome DevTools MCP shipped — browser automation now standard MCP tool | MCP ecosystem | Positioned as governance story, not browser story. |
|
||||
@ -81,12 +84,12 @@ Track competitor releases and market events that affect Phase 30 positioning. En
|
||||
## Competitor Feature Tracker
|
||||
|
||||
### LangGraph
|
||||
- A2A support: **VERIFY** — PRs #6645, #7113, #7205 not found as open PRs in langchain-ai/langgraph. Either merged/closed or re-numbered. Requires manual re-check. Last confirmed: 2026-04-21 cycle.
|
||||
- A2A support: **OPEN** — PRs #6645, #7113, #7205 still OPEN in langchain-ai/langgraph as of 2026-04-23T17:38Z. Live production claim intact. Expected GA: Q2-Q3 2026.
|
||||
- Graph orchestration: ✅ Live
|
||||
- HiTL workflows: **VERIFY** — recent streaming and subgraph PRs (#7559, #7550) do not appear to be HiTL; re-verify
|
||||
- Self-hosted enterprise: ❌ SaaS-only via LangGraph Studio
|
||||
- Marketplace: ❌ None
|
||||
- Source: GitHub langchain-ai/langgraph (verified 2026-04-21 20:35Z) — PRs #6645, #7113, #7205 not found. Recommend manual re-check.
|
||||
- Source: GitHub langchain-ai/langgraph (verified 2026-04-23 17:38Z) — PRs #6645, #7113, #7205 confirmed OPEN.
|
||||
|
||||
### CrewAI
|
||||
- External agent support: ✅ Secondary path
|
||||
@ -115,7 +118,7 @@ Track competitor releases and market events that affect Phase 30 positioning. En
|
||||
- **Check frequency:** Every marketing cycle
|
||||
- **Trigger:** Any competitor shipping something that invalidates a Phase 30 positioning claim
|
||||
- **File location:** `docs/ecosystem-watch.md` (origin/main)
|
||||
- **Last updated by:** PMM | 2026-04-21
|
||||
- **Last updated by:** PMM | 2026-04-23 (LangGraph PRs verified OPEN; new feat PRs #1730/#1702/#1731 logged; release note written)
|
||||
|
||||
---
|
||||
|
||||
|
||||
130
docs/engineering/postmortem-2026-04-23-boot-event-401.md
Normal file
130
docs/engineering/postmortem-2026-04-23-boot-event-401.md
Normal file
@ -0,0 +1,130 @@
|
||||
# Incident: SaaS tenant provisioning 401 on /cp/tenants/boot-event
|
||||
|
||||
**Date:** 2026-04-23
|
||||
**Severity:** High — every new SaaS tenant blocked
|
||||
**Detection path:** E2E Staging SaaS run 24848425822 failed at "tenant provisioning"; investigation of CP Railway logs surfaced the auth mismatch.
|
||||
**Status:** Fix pushed on [molecule-controlplane#238](https://github.com/Molecule-AI/molecule-controlplane/pull/238).
|
||||
**Related:** [issue #239](https://github.com/Molecule-AI/molecule-controlplane/issues/239) (Cloudflare DNS record quota), [testing-strategy.md](../engineering/testing-strategy.md)
|
||||
|
||||
## Summary
|
||||
|
||||
For ~3 days leading up to 2026-04-23, every new SaaS tenant failed to transition from `provisioning` → `running`. The EC2 instance would boot, read its `admin_token` from AWS Secrets Manager, and attempt to POST `/cp/tenants/boot-event` on the control plane. Every request got 401 Unauthorized. Without a successful boot event, CP would wait 4 minutes, fall through to a canary probe (which also failed due to an unrelated Cloudflare DNS quota issue), and write a `status='failed'` row. The tenant would then be stuck forever.
|
||||
|
||||
## Root cause
|
||||
|
||||
**A race between EC2 boot and the DB write of `org_instances.admin_token`.**
|
||||
|
||||
The flow was:
|
||||
|
||||
1. CP `provisionTenant()` called `Provision()`, which:
|
||||
- Generated `admin_token = generatePassword()`
|
||||
- Wrote it to AWS Secrets Manager
|
||||
- Returned it in the `Result` struct
|
||||
2. EC2 launched in parallel; user-data started running.
|
||||
3. **Before** CP's `provisionTenant()` wrote the `org_instances` row, it called `WaitForTenantReady()` — a 4-minute poll of `org_instance_boot_events`.
|
||||
4. EC2 finished its early boot stages (~60-90s) and started POSTing `/cp/tenants/boot-event` with the `admin_token` from Secrets Manager.
|
||||
5. CP's inline auth on that endpoint does:
|
||||
```sql
|
||||
SELECT org_id FROM org_instances WHERE admin_token = $1 AND admin_token != ''
|
||||
```
|
||||
No row existed yet. → 401.
|
||||
6. Every subsequent boot-event post: 401.
|
||||
7. `WaitForTenantReady` saw no events (because 401s never write to `org_instance_boot_events`). After 4 minutes it returned `false`.
|
||||
8. Fell through to canary. Canary failed (unrelated — Cloudflare DNS quota exceeded, so the tenant's hostname didn't resolve).
|
||||
9. `insertFailedInstance` wrote a row **without** `admin_token`. Tenant stuck in `failed`.
|
||||
|
||||
### The commit that introduced the bug
|
||||
|
||||
[molecule-controlplane#235](https://github.com/Molecule-AI/molecule-controlplane/pull/235) — "fix(provision): wait for tenant boot-event before falling back to canary". Merged 2026-04-22.
|
||||
|
||||
Before #235, readiness was determined via a canary probe through Cloudflare's edge — which didn't need CP-side auth, so the INSERT ordering didn't matter. #235 made boot-events the primary readiness signal but didn't move the INSERT earlier. The race was latent before but became load-bearing after.
|
||||
|
||||
## Detection
|
||||
|
||||
**What should have caught it:**
|
||||
|
||||
- ❌ Unit tests on `provisionTenant` — existed, but they used `fakeProv` and `noopCanaryOK` that bypassed the real auth flow. They asserted the INSERT happened eventually; they didn't assert the INSERT happened *before* boot-event auth.
|
||||
- ❌ Integration tests — CP has no end-to-end integration test that provisions a real tenant with real auth against a real DB. The E2E Staging SaaS flow is the closest, and it only ran in CI after merge.
|
||||
- ✅ E2E Staging SaaS — did catch it, but ~20 hours after merge. Blast radius by then: every new tenant in staging, including all E2E runs.
|
||||
|
||||
**What actually caught it:**
|
||||
|
||||
Manual investigation of CP Railway logs for the failed E2E run. Grepping for the tenant org_id + examining the `[GIN] POST /cp/tenants/boot-event` status codes revealed the 401 pattern.
|
||||
|
||||
## Timeline
|
||||
|
||||
| Time (UTC) | Event |
|
||||
|---|---|
|
||||
| 2026-04-22 ~late | PR #235 merged to controlplane main — introduces the race |
|
||||
| 2026-04-22 → 23 | Nightly E2E Staging SaaS fails (no alert wired) |
|
||||
| 2026-04-23 07:14 | E2E on main also fails with the same signature |
|
||||
| 2026-04-23 morning | Investigation starts; misattributed to hermes provider 401 (separate known bug) |
|
||||
| 2026-04-23 17:09 | Fresh E2E run 24848425822 dispatched on staging sha `6539908` |
|
||||
| 2026-04-23 17:13 | Run fails with "tenant provisioning failed" |
|
||||
| 2026-04-23 ~17:15 | Railway logs inspection reveals the 401s on `/cp/tenants/boot-event` |
|
||||
| 2026-04-23 17:30 | Root cause identified — admin_token not in DB when EC2 phones home |
|
||||
| 2026-04-23 ~17:50 | Fix pushed on controlplane `fix/provision-readiness-boot-events` |
|
||||
| 2026-04-23 ~18:00 | PR #238 opened, CI running |
|
||||
|
||||
## Fix
|
||||
|
||||
Write the `org_instances` row with `status='provisioning'` and `admin_token` **immediately after** `Provision()` returns, **before** `WaitForTenantReady()`. Flip `status='running'` once readiness passes.
|
||||
|
||||
```go
|
||||
// NEW: early INSERT so boot-events can authenticate
|
||||
if _, err := h.db.ExecContext(ctx, `
|
||||
INSERT INTO org_instances (org_id, ..., admin_token, status)
|
||||
VALUES ($1, ..., $8, 'provisioning')
|
||||
ON CONFLICT (org_id) DO UPDATE SET ..., status = 'provisioning'
|
||||
`, ...); err != nil {
|
||||
h.insertFailedInstance(ctx, org.ID, ...)
|
||||
return
|
||||
}
|
||||
|
||||
// THEN wait for readiness — boot-events will now authenticate
|
||||
bootReady, _ := provisioner.WaitForTenantReady(ctx, h.db, org.ID, 4*time.Minute)
|
||||
|
||||
// ... canary fallback as before ...
|
||||
|
||||
// Finally, transition to 'running'
|
||||
h.db.ExecContext(ctx, `UPDATE org_instances SET status = 'running' WHERE org_id = $1`, org.ID)
|
||||
```
|
||||
|
||||
See [molecule-controlplane#238](https://github.com/Molecule-AI/molecule-controlplane/pull/238) for the full diff.
|
||||
|
||||
## Lessons
|
||||
|
||||
### 1. "Write state before dependent reads" is a general pattern
|
||||
|
||||
The same chicken-and-egg shape applies anywhere a newly-provisioned entity phones home for its own state. Future auth-gated callbacks should follow the rule: **persist the credential in the validation store BEFORE the entity can call back with it.** Include in code review checklist for provisioning-adjacent changes.
|
||||
|
||||
### 2. Unit tests that use fakes can't catch auth-flow races
|
||||
|
||||
The existing `TestProvisionTenant_*` tests used `fakeProv` and `noopCanaryOK` that elided the real auth check. They asserted the shape of DB writes but not the temporal ordering relative to an external caller's expectation. For provisioning flows specifically, we need an integration-test tier that exercises real HTTP → real DB with the actual auth middleware.
|
||||
|
||||
**Action:** Add a CP integration-test target (`make test-integration`) that spins up a real Postgres + CP binary + a fake EC2 that mimics user-data's boot-event POST cadence. File as follow-up.
|
||||
|
||||
### 3. E2E failures need faster detection
|
||||
|
||||
E2E Staging SaaS failed silently overnight. Nobody knew until someone manually ran `gh run list` and saw the red dots. The alert latency from merge to awareness was ~20 hours.
|
||||
|
||||
**Action:** Wire E2E Staging SaaS failures to a push notification or Telegram alert channel. File as follow-up.
|
||||
|
||||
### 4. Code comments should describe invariants, not the happy path
|
||||
|
||||
The `provisionTenant` function had comments describing what each block did, but nothing stating **"this function must write `org_instances.admin_token` before any code path that triggers an external callback using it."** If that invariant had been written down, the #235 author would likely have noticed the ordering change broke it.
|
||||
|
||||
**Action:** When landing this fix, add the invariant to a doc comment at the top of `provisionTenant`.
|
||||
|
||||
### 5. Separate unrelated failures — don't conflate
|
||||
|
||||
Early investigation blamed the hermes provider 401 bug (a separate, known issue affecting hermes-agent startup after tenant came up). Those 401s come from `hermes-agent error 401` in the workspace-server logs, not from CP Railway logs. Two different 401s with totally different causes. **When debugging, always check which component is emitting the 401 before assuming it's the known one.**
|
||||
|
||||
## Follow-ups
|
||||
|
||||
- [ ] Land [molecule-controlplane#238](https://github.com/Molecule-AI/molecule-controlplane/pull/238)
|
||||
- [ ] Redeploy staging-api, verify E2E goes green
|
||||
- [ ] Add CP integration test suite (see lesson #2)
|
||||
- [ ] Wire E2E failure → notification (see lesson #3)
|
||||
- [ ] Add invariant comment in `provisionTenant` (see lesson #4)
|
||||
- [ ] Cloudflare DNS quota cleanup — [molecule-controlplane#239](https://github.com/Molecule-AI/molecule-controlplane/issues/239)
|
||||
142
docs/engineering/pr-hygiene.md
Normal file
142
docs/engineering/pr-hygiene.md
Normal file
@ -0,0 +1,142 @@
|
||||
# Pull Request Hygiene
|
||||
|
||||
**Status:** Guide. Violations are a review-time flag, not a CI gate.
|
||||
**Audience:** Humans and agents opening PRs in this repo.
|
||||
**Cross-refs:** [testing-strategy.md](./testing-strategy.md), [backends.md](../architecture/backends.md)
|
||||
|
||||
## Why this exists
|
||||
|
||||
On 2026-04-23 a backlog audit found **23 open PRs on molecule-core**, of which 8 had accumulated 70-380 files of bloat (+2000/-8000 lines) from stale branch drift. The underlying fix in each was 1-5 files; the rest was merge artifact. Half the PRs were closed that day because they weren't reviewable and the real fix had to be re-extracted onto a clean branch.
|
||||
|
||||
This document captures the patterns that avoid that outcome.
|
||||
|
||||
## The rules
|
||||
|
||||
### 1. Small PRs, single concern
|
||||
|
||||
| Change size | Reviewability |
|
||||
|---|---|
|
||||
| ≤100 lines | ✅ Good. One sitting. |
|
||||
| 100-300 lines | ⚠️ Acceptable if genuinely one logical change. |
|
||||
| 300-1000 lines | 🔴 Too large. Split. |
|
||||
| 1000+ lines | 🚫 Unreviewable — split before opening. |
|
||||
|
||||
**Exception:** complete file deletions and automated refactors where the reviewer only needs to verify intent.
|
||||
|
||||
### 2. Branch hygiene — rebase, don't merge-in
|
||||
|
||||
When your branch falls behind the base:
|
||||
|
||||
**Do:**
|
||||
```bash
|
||||
git fetch origin staging
|
||||
git rebase origin/staging
|
||||
# resolve conflicts
|
||||
git push --force-with-lease
|
||||
```
|
||||
|
||||
**Don't:**
|
||||
```bash
|
||||
git fetch origin staging
|
||||
git merge origin/staging # creates merge commit + pulls ALL of base's files into your diff
|
||||
```
|
||||
|
||||
A merge commit from `origin/staging` brings every base-branch commit into your PR's diff. That's where the 235-file bloat comes from. Once you have it, you can't get rid of it without resetting the branch.
|
||||
|
||||
### 3. If your branch has already drifted — cherry-pick onto fresh base
|
||||
|
||||
```bash
|
||||
# Identify your real commits
|
||||
git log origin/staging..HEAD
|
||||
|
||||
# Create a fresh branch off current base
|
||||
git checkout -b your-branch-clean origin/staging
|
||||
|
||||
# Cherry-pick only the commits you actually authored
|
||||
git cherry-pick abc1234 def5678
|
||||
|
||||
# Push and open a new PR; close the old one as "superseded by #N"
|
||||
git push -u origin your-branch-clean
|
||||
```
|
||||
|
||||
**Don't** try to rebase a drifted branch interactively to remove the base-branch commits. It fights you every merge.
|
||||
|
||||
### 4. Target `staging` unless you're doing a staging→main promote
|
||||
|
||||
Per branching policy ([feedback memory](../../.claude/projects/-Users-hongming-Documents-GitHub-molecule-monorepo/memory/feedback_no_push_main.md) rule): every change lands on `staging` first. Once validated there, a periodic `chore: sync staging → main` PR promotes the bundle.
|
||||
|
||||
Exception: hotfixes that also land on `main` directly with CEO approval.
|
||||
|
||||
### 5. Describe the why, not the what
|
||||
|
||||
A good PR title:
|
||||
- `fix(provision): write org_instances row BEFORE readiness check to unblock boot-event auth`
|
||||
|
||||
A bad PR title:
|
||||
- `Update orgs.go`
|
||||
- `Fix bug`
|
||||
- `Phase 1`
|
||||
|
||||
The body should explain:
|
||||
- **What's broken / missing** (or what's the opportunity)
|
||||
- **Why this fix** — especially if there are alternatives you considered
|
||||
- **What's tested** — which scenarios the test plan covers
|
||||
- **What's deferred** — if there are follow-ups, file issues and link them
|
||||
|
||||
Anti-pattern: `## Summary\n- Fix bug`. That's not a summary; that's a stub.
|
||||
|
||||
### 6. Close the loop on review comments
|
||||
|
||||
- Comments labeled `Nit:` / `Optional:` / `FYI` can be left for follow-up — but leave a reply acknowledging.
|
||||
- Critical/required comments need a fix or a justified reply before merge.
|
||||
- Don't resolve threads without replying — silent resolves read as dismissal.
|
||||
|
||||
### 7. CI must be green (or the failure must be acknowledged)
|
||||
|
||||
- Never push `--no-verify` unless explicitly requested.
|
||||
- If a pre-existing failure is blocking merge, document it inline and file a tracking issue — don't silently let it erode the "all green" norm.
|
||||
|
||||
## Patterns for specific situations
|
||||
|
||||
### Re-targeting an old branch
|
||||
|
||||
When a PR was opened weeks ago against `main` but policy now says `staging`:
|
||||
|
||||
```bash
|
||||
git fetch origin staging
|
||||
git rebase --onto origin/staging old-base HEAD
|
||||
git push --force-with-lease
|
||||
# Edit the PR's base branch in GitHub UI
|
||||
```
|
||||
|
||||
### Splitting a large PR
|
||||
|
||||
If your PR is already open and the reviewer asks for a split:
|
||||
|
||||
1. Identify the cleanest split boundary — usually along file groups or dependency layers.
|
||||
2. Create two new branches off current staging.
|
||||
3. Cherry-pick the commits for each concern into its branch.
|
||||
4. Open two new PRs, close the original as "superseded by #A and #B".
|
||||
|
||||
### Marketing / docs-heavy PRs
|
||||
|
||||
Marketing content has been moved to an internal repo per commit `93324e7`. If your PR modifies files under `docs/marketing/campaigns/`, `docs/marketing/plans/`, or `docs/marketing/briefs/` (with non-public-facing strategy content):
|
||||
|
||||
1. Check if the file still exists on `origin/staging`.
|
||||
2. If deleted, open the PR in the internal marketing repo instead.
|
||||
3. Public-facing marketing (blog posts, SEO pages under `docs/blog/`) stays in this repo.
|
||||
|
||||
## Signs your PR has a hygiene problem
|
||||
|
||||
- **70+ files changed** when your commit message mentions 2-3 files
|
||||
- **+2000/-3500 lines** but the actual fix is ~100 lines
|
||||
- **State: DIRTY** in GitHub for >1 day
|
||||
- Filenames in the diff you don't recognize (someone else's changes in your PR)
|
||||
- Merge commits in your branch's log named `Merge remote-tracking branch 'origin/staging' into ...`
|
||||
|
||||
If you see any of these, don't try to "clean it up in place" — **cherry-pick onto a fresh branch** (rule 3 above).
|
||||
|
||||
## Related
|
||||
|
||||
- [Issue #1822](https://github.com/Molecule-AI/molecule-core/issues/1822) — backend parity drift tracker (example of docs that have to stay current)
|
||||
- [Postmortem: CP boot-event 401](./postmortem-2026-04-23-boot-event-401.md) — caught before shipping because a reviewer could read the diff
|
||||
111
docs/engineering/testing-strategy.md
Normal file
111
docs/engineering/testing-strategy.md
Normal file
@ -0,0 +1,111 @@
|
||||
# Testing Strategy
|
||||
|
||||
**Status:** Policy. Update when tier definitions or thresholds change.
|
||||
**Audience:** Everyone writing or reviewing code in this repo.
|
||||
**Cross-refs:** [backends.md](../architecture/backends.md), [pr-hygiene.md](./pr-hygiene.md), [postmortem-2026-04-23-boot-event-401.md](./postmortem-2026-04-23-boot-event-401.md)
|
||||
|
||||
## The short version
|
||||
|
||||
- **Don't chase 100% coverage.** The last 15-20% costs as much as the first 80% and mostly adds brittle tests of trivial getters, error branches that can't fire, and stdlib wrappers.
|
||||
- **Different code classes have different floors.** Auth at 80% is scarier than a DTO at 50%. Match the test investment to the risk.
|
||||
- **Tests should pay rent.** A test that runs lines but asserts nothing meaningful isn't catching bugs — it's just dragging refactors down.
|
||||
|
||||
## Tiered coverage floors
|
||||
|
||||
Every Go package, every TypeScript module, every Python module fits one of these tiers. The tier determines the minimum acceptable coverage — and the review standard.
|
||||
|
||||
| Tier | Examples | Line floor | Branch floor | Review standard |
|
||||
|---|---|---|---|---|
|
||||
| **1. Auth / secrets / crypto** | `tokens`, `session_auth`, `wsauth_middleware`, `crypto/envelope`, `cp_tenant_auth` | **90%** | **85%** | Every branch tested. Adversarial scenarios (cross-tenant, expired token, null origin, malformed header). Timing considered. |
|
||||
| **2. Handlers with side effects** | `workspace_provision`, `workspace_crud`, `container_files`, `terminal`, `registry` | **75%** | 70% | Happy + main error paths. DB mocks. Ownership / tenant-isolation checks. |
|
||||
| **3. State machines + workers** | `scheduler`, `provisioner`, `healthsweep`, `orphan-sweeper`, `boot_ready` | **75%** | 70% | Every state transition tested, plus the transitions that *shouldn't* fire. |
|
||||
| **4. Config / business logic** | `budget`, `orgtoken` (validation), `templates`, `derive-provider`, `redaction` | **70%** | 65% | Standard unit-test territory. Table-driven preferred. |
|
||||
| **5. Plain DTOs / generated** | `models/*`, proto-generated Go, TypeScript interfaces | none | none | Writing tests here is theatre. Don't. |
|
||||
| **6. CLI glue / cmd/*** | `cmd/server`, `cmd/molecli` | smoke only | — | Integration tests / E2E cover these. One startup-smoke test per binary. |
|
||||
| **7. Third-party wrappers** | `awsapi`, `cloudflareapi`, `stripeapi`, `neonapi` | integration | — | Unit tests mock vendor shape, not behavior. Real behavior covered by staging integration. |
|
||||
|
||||
### Why a blanket percentage is wrong
|
||||
|
||||
- A `models/` package at 90% means you wrote tests for `func (w Workspace) ID() string { return w.id }`. No bugs caught, but coverage number is green.
|
||||
- A `tokens` package at 75% means some rejection branch isn't covered. Maybe the *exact* branch that lets a revoked token still authenticate.
|
||||
- Blanket targets make the first case look equivalent to the second. They aren't.
|
||||
|
||||
## Current state (as of 2026-04-23)
|
||||
|
||||
Run `go test ./... -cover` in each repo for up-to-date numbers. Snapshot:
|
||||
|
||||
### workspace-server (Go)
|
||||
|
||||
| Package | Actual | Tier | Target | Gap |
|
||||
|---|---:|---|---:|---:|
|
||||
| `internal/handlers/tokens.go` | **0%** | 1 | 90% | 90 |
|
||||
| `internal/handlers/workspace_provision.go` | **0%** | 2 | 75% | 75 |
|
||||
| `internal/middleware/wsauth_middleware.go` | ~48% | 1 | 90% | 42 |
|
||||
| `internal/provisioner` | 45% | 3 | 75% | 30 |
|
||||
| `internal/scheduler` | 49% | 3 | 75% | 26 |
|
||||
| `internal/channels` | 40% | 4 | 70% | 30 |
|
||||
| `internal/orgtoken` | 88% | 4 | 70% | — |
|
||||
| `internal/crypto` | 91% | 1 | 90% | — |
|
||||
| `internal/supervised` | 93% | 3 | 75% | — |
|
||||
| `internal/plugins` | 94% | 4 | 70% | — |
|
||||
| `internal/envx` | 100% | 5 | none | — |
|
||||
|
||||
### molecule-controlplane (Go)
|
||||
|
||||
| Package | Actual | Tier | Target | Gap |
|
||||
|---|---:|---|---:|---:|
|
||||
| `internal/awsapi` | 18% | 7 | integration | — |
|
||||
| `internal/provisioner` | 48% | 3 | 75% | 27 |
|
||||
| `internal/handlers` | 60% | 2 | 75% | 15 |
|
||||
| `internal/billing` | 60% | 4 | 70% | 10 |
|
||||
| `internal/crypto` | 68-80% | 1 | 90% | 10-22 |
|
||||
| `internal/auth` | 96% | 1 | 90% | — |
|
||||
| `internal/middleware` | 97% | 1 | 90% | — |
|
||||
| `internal/reserved` | 100% | 5 | none | — |
|
||||
| `internal/httpx` | 100% | 4 | 70% | — |
|
||||
|
||||
### canvas (TypeScript)
|
||||
|
||||
**No coverage instrumentation today.** 900 tests / 58 files pass, but coverage isn't measured. See issue #1815 for the fix: set a 70% line floor in `vitest.config.ts` and gate CI on it.
|
||||
|
||||
### workspace (Python)
|
||||
|
||||
**No pytest/coverage config.** See issue #1818: set up `pytest-cov` with `--cov-fail-under=75` (ratchet from current baseline over 2-3 weeks).
|
||||
|
||||
## Writing a good test
|
||||
|
||||
A good test:
|
||||
- **Asserts a specific outcome**, not that a function runs without error.
|
||||
- **Covers the exact branch that bugs would live in** — cross-tenant access, revoked-but-cached token, race on state transition.
|
||||
- **Uses table-driven patterns** when the code is a dispatch with N cases. One test row per case.
|
||||
- **Mocks at system boundaries** (DB, HTTP, time), not at internal package boundaries.
|
||||
- **Survives refactors** — tests behavior, not internal state.
|
||||
|
||||
A bad test:
|
||||
- Tests a getter that just returns a field.
|
||||
- Mocks the function under test itself.
|
||||
- Relies on `time.Sleep` or clock timing to assert order.
|
||||
- Asserts `nil == nil` to boost coverage.
|
||||
|
||||
## Enforcement
|
||||
|
||||
### CI gates
|
||||
|
||||
- **Go**: `go test ./... -cover` + a pre-commit script that compares coverage to `.coverage-baseline` and fails on drops > 2 points in a tier-1 package.
|
||||
- **TypeScript**: `vitest --coverage` with thresholds in `vitest.config.ts`. Fails CI if below.
|
||||
- **Python**: `pytest --cov-fail-under=75` in the Python CI job.
|
||||
|
||||
### Review expectations
|
||||
|
||||
- Any PR touching a tier-1 package that lowers its coverage needs an explicit reviewer sign-off and justification.
|
||||
- New code should arrive at or above its tier's floor.
|
||||
- Untested files in tier-1 or tier-2 should be flagged in review, not waved through.
|
||||
|
||||
## Related
|
||||
|
||||
- [Issue #1821](https://github.com/Molecule-AI/molecule-core/issues/1821) — policy tracking issue
|
||||
- [Issue #1815](https://github.com/Molecule-AI/molecule-core/issues/1815) — Canvas coverage instrumentation
|
||||
- [Issue #1818](https://github.com/Molecule-AI/molecule-core/issues/1818) — Python pytest-cov
|
||||
- [Issue #1814](https://github.com/Molecule-AI/molecule-core/issues/1814) — workspace_provision_test.go unblock
|
||||
- [Issue #1816](https://github.com/Molecule-AI/molecule-core/issues/1816) — tokens.go coverage
|
||||
- [Issue #1819](https://github.com/Molecule-AI/molecule-core/issues/1819) — wsauth_middleware coverage
|
||||
44
docs/marketing/blog/2026-04-23-saas-file-api-fix.md
Normal file
44
docs/marketing/blog/2026-04-23-saas-file-api-fix.md
Normal file
@ -0,0 +1,44 @@
|
||||
# SaaS Workspaces Now Support Full File API — SSH-Backed Writes Land Today
|
||||
|
||||
**Status:** Live — merged 2026-04-23
|
||||
**PR:** [#1702](https://github.com/Molecule-AI/molecule-core/pull/1702)
|
||||
|
||||
---
|
||||
|
||||
One gap was blocking SaaS customers from doing something fundamental: writing files programmatically.
|
||||
|
||||
When you called `PUT /workspaces/:id/files/config.yaml` from a SaaS (EC2-backed) workspace, you got a 500. `failed to write file: docker not available`. The file API existed, but only for self-hosted Docker deployments. SaaS workspaces — the ones running on real EC2 VMs — had no path to write.
|
||||
|
||||
That changes today.
|
||||
|
||||
## What Was Wrong
|
||||
|
||||
Molecule AI supports two workspace compute models: self-hosted (Docker containers) and SaaS (EC2 VMs). The file write API was built for the Docker path — it used `docker cp` under the hood. SaaS workspaces don't have Docker. There was no fallback, so every API write failed silently.
|
||||
|
||||
This wasn't a permissions issue or a timeout. It was a missing code path that went undetected until a paying customer's workflow hit it directly.
|
||||
|
||||
## What's Fixed
|
||||
|
||||
The file write API now detects which compute model is in use and routes accordingly:
|
||||
|
||||
- **Self-hosted (Docker):** Unchanged — `docker cp` path still used
|
||||
- **SaaS (EC2):** Routes through EC2 Instance Connect (EIC) — the same ephemeral-keypair SSH flow that powers the Terminal tab in the Canvas
|
||||
|
||||
The remote write uses `install -m 0644 /dev/stdin <path>` for an atomic write that creates missing parent directories. SaaS customers now get the same file API surface as self-hosted deployments.
|
||||
|
||||
## Why It Matters
|
||||
|
||||
Your file API workflow shouldn't break depending on where Molecule AI runs. Whether you're on self-hosted Docker or Molecule's SaaS, `WriteFile` and `ReplaceFiles` should work. They do now.
|
||||
|
||||
**Try it:**
|
||||
```bash
|
||||
curl -X PUT https://your-workspace.moleculesai.app/workspaces/:id/files/config.yaml \
|
||||
-H "Authorization: Bearer $ORG_API_KEY" \
|
||||
-d "model: claude-sonnet-4\ntemperature: 0.7"
|
||||
```
|
||||
|
||||
File API. Now everywhere Molecule AI runs.
|
||||
|
||||
---
|
||||
|
||||
*Found a bug or have a feature request? Open an issue at [github.com/Molecule-AI/molecule-core](https://github.com/Molecule-AI/molecule-core).*
|
||||
@ -0,0 +1,82 @@
|
||||
# PR #1686 Positioning Brief: Tool Trace + Platform Instructions
|
||||
|
||||
**Source:** PR #1686 — `feat: tool trace + platform instructions`
|
||||
**Date:** 2026-04-23
|
||||
**Author:** PMM
|
||||
**Status:** Draft — for internal review before announcement
|
||||
|
||||
---
|
||||
|
||||
## Target Buyer
|
||||
|
||||
**Primary:** Platform Engineering / DevOps leads (80% of value)
|
||||
**Secondary:** Enterprise IT / Security Governance leads (Platform Instructions)
|
||||
|
||||
Platform teams own the agent runtime and are the first to get paged when an agent goes off-script. They need built-in observability, not bolt-on stitching. Enterprise IT and compliance teams care about the governance angle — system-prompt rules that enforce behavior before an agent runs, not after it has already done something unintended.
|
||||
|
||||
---
|
||||
|
||||
## Primary Value Prop
|
||||
|
||||
> **Tool Trace** gives every A2A response a complete, run_id-paired execution record — so platform teams can trace what every agent actually did, without wiring up a third-party SDK.
|
||||
|
||||
> **Platform Instructions** lets workspace admins enforce system-prompt rules at startup — so governance happens before the agent runs, not after an incident.
|
||||
|
||||
---
|
||||
|
||||
## Competitive Angle
|
||||
|
||||
**vs. Langfuse / Helicone / separate observability pipelines:**
|
||||
Third-party LLM observability tools require instrumentation in every agent: SDK installs, API key management, proxy configuration, and a separate vendor relationship. Tool Trace ships the execution record inside every A2A message and stores it in `activity_logs` — no extra pipeline, no separate pane of glass. For teams already on Molecule, it's zero-lift observability.
|
||||
|
||||
Langfuse/Helicone remain stronger for *cross-platform, multi-model* observability (tracking OpenAI + Anthropic + self-hosted in one view). That's not Molecule's fight. The positioning here is: "If you're already running agents on Molecule, you already have enterprise-grade trace — turn it on, don't integrate it."
|
||||
|
||||
**vs. Hermes native tool tracing:**
|
||||
Hermes traces individual model calls. Tool Trace traces *agent behavior* — the A2A-level sequence of tool calls and responses across the full task lifecycle. Different layer of the stack. Tool Trace is additive, not competitive.
|
||||
|
||||
**vs. policy-as-code tools (OPA, Sentinel):**
|
||||
Platform Instructions enforces behavioral guardrails at the system-prompt level. Policy engines enforce runtime resource access. They complement; Platform Instructions is earlier in the chain (pre-execution vs. during-execution).
|
||||
|
||||
---
|
||||
|
||||
## Key Differentiator
|
||||
|
||||
Tool Trace and Platform Instructions are **platform-native** — not plugins, not third-party SDKs, not configuration-as-code you have to maintain. They live where the agent runs: inside the workspace startup path and inside every A2A message envelope. There's nothing to install, no API key to rotate, no version drift to manage when the agent framework updates.
|
||||
|
||||
Third-party observability and governance tooling always has a lag between "agent framework ships a new behavior" and "our integration captures it." Native trace and prompt-level instructions have no lag — they are the platform.
|
||||
|
||||
---
|
||||
|
||||
## Objection Handlers
|
||||
|
||||
**O1: "We already use Datadog / Langfuse / Splunk for this."**
|
||||
That's fine for cross-platform, multi-model environments. Tool Trace captures *A2A-level* agent behavior — tool calls, input/output previews, run_id-paired sequences — that generic LLM observability pipelines typically miss or flatten. Think of it as your Molecule-specific layer inside your existing observability stack. It doesn't replace Datadog; it enriches it.
|
||||
|
||||
**O2: "Why enforce system-prompt rules at the platform level instead of in code?"**
|
||||
Because code changes require a deployment, and governance that requires a deployment is governance that only happens at the next release cycle. Platform Instructions are workspace-scoped rules that take effect at startup — a platform team or IT admin can update agent behavior without touching application code or triggering a redeploy. Speed of governance matters.
|
||||
|
||||
---
|
||||
|
||||
## Overlap / Conflict Notes
|
||||
|
||||
| Existing Feature | Relationship |
|
||||
|-----------------|--------------|
|
||||
| Org-scoped API keys (#1105) | Different layer: API key auth vs. agent behavior/prompt. Tool Trace traces what agents *do* with the keys; org keys control *who gets* the keys. Not cannibalization — complementary. |
|
||||
| Audit trail visualization panel (#759) | Tool Trace is the raw execution record; the audit trail panel is the compliance UI on top of it. Tool Trace feeds the audit trail. Not competitive — dependency. |
|
||||
| Snapshot secret scrubber (#977) | Both platform observability. Secret scrubber is about data posture; Tool Trace is about behavior. No conflict. |
|
||||
|
||||
**Cannibalization risk: LOW.** Tool Trace and Platform Instructions occupy the observability/governance vertical that existing features touch from different angles — no direct overlap, strong adjacency.
|
||||
|
||||
---
|
||||
|
||||
## CTA
|
||||
|
||||
**For platform teams:** "Enable activity log tracing for your workspace — every A2A task now has a complete execution record, no SDK required."
|
||||
**For enterprise IT:** "Set workspace-level system prompt rules to enforce behavioral guardrails before agents run. No code deploy required."
|
||||
**Combined anchor:** "Molecule gives you observability and governance as platform primitives — not afterthought integrations."
|
||||
|
||||
---
|
||||
|
||||
## Recommended Announcement Angle
|
||||
|
||||
Lead with the platform-native story, not the feature list. The headline is: *"Molecule agents now come with built-in execution tracing and governance — nothing to integrate."* Avoid leading with "Tool Trace" as a feature name in top-level copy; use "execution tracing" or "agent observability" for broader appeal.
|
||||
115
docs/marketing/briefs/cloudflare-artifacts-positioning.md
Normal file
115
docs/marketing/briefs/cloudflare-artifacts-positioning.md
Normal file
@ -0,0 +1,115 @@
|
||||
# Cloudflare Artifacts — PMM Positioning Brief
|
||||
**Source:** PR #641, merged 2026-04-17 | Blog: `docs/marketing/blog/2026-04-21-cloudflare-artifacts-integration.md`
|
||||
**Issue:** #1174 | **Status:** PMM DRAFT | **Date:** 2026-04-23
|
||||
**Owner:** PMM | **Blocking:** none — feature shipped, ready for social
|
||||
|
||||
---
|
||||
|
||||
## Positioning Decision
|
||||
|
||||
**Use "Git for agents" as the headline metaphor — with qualification.**
|
||||
|
||||
Cloudflare's own beta announcement uses "Git for agents." It's the right hook because developers immediately understand what it means and why it matters. Leading with it is accurate and immediately differentiating.
|
||||
|
||||
The qualification: this is Git *plus* the agent primitives that make it agent-native. Automated commits (no human in the loop), API-first branching, ephemeral short-lived credentials, canvas-native integration. It's not Git with a chat interface — it's version control designed for stateless agents.
|
||||
|
||||
**Recommended headline:** "Give your agents a Git history — without touching a terminal."
|
||||
|
||||
---
|
||||
|
||||
## Buyer Profile
|
||||
|
||||
**Primary:** Platform engineers and DevOps leads evaluating AI agent platforms. They have agents running in production, they're managing agent state manually or not at all, and they need version control they can instrument. They're not necessarily Git experts — they're the people who inherited the AI agent rollout.
|
||||
|
||||
**Secondary:** Enterprise security and compliance teams. They need audit trails on agent actions. A versioned snapshot system with immutable commits is a concrete answer to "what did the agent change?" — without requiring agents to write human-readable commit messages.
|
||||
|
||||
**Not the audience:** Developers who want Git workflows in their own IDE. This isn't replacing GitHub for human developers — it's giving agents a version history that humans can audit and roll back.
|
||||
|
||||
---
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Use Case 1: Multi-agent pipelines without manual handoff
|
||||
Two agents, same task. Agent A writes a feature branch. Agent B reviews and approves. You merge. No Slack threads asking "did the research agent finish?" No copy-pasting outputs between workspaces.
|
||||
|
||||
### Use case 2: Crash recovery without starting over
|
||||
An agent crashes mid-task. With versioned snapshots, the last checkpoint is a Git commit. The next agent to pick up the task starts from a diff, not a blank workspace.
|
||||
|
||||
### Use case 3: Experimentation without risk
|
||||
Agents trying something risky can fork a branch first. If it fails, delete the fork. The main branch is clean. No "oops, can you revert that?" in the team Slack.
|
||||
|
||||
---
|
||||
|
||||
## Top 2 Buyer Objections
|
||||
|
||||
### Objection 1: "Why not just use GitHub? Agents can call `git commit`"
|
||||
**Likely buyer:** Platform engineers with existing GitOps workflows.
|
||||
|
||||
**The problem with this objection:** `git commit` requires a Git repo on disk, human-readable messages, and a human in the loop to resolve conflicts. Agents don't naturally produce well-structured commits. And "just use GitHub" means agents need credentials, network access, and a configured remote — which creates a dependency you have to manage.
|
||||
|
||||
**Recommended response:**
|
||||
Git was designed for humans. Agents need version control that works without a human in the commit loop — automatic snapshots, API-first branching, ephemeral credentials that never get stored. Cloudflare Artifacts gives agents their own versioned storage without requiring Git credentials on every agent instance. The four API operations (`POST /artifacts/repos`, `fork`, `import`, `tokens`) are agent-native — no terminal, no commit messages, no credential management.
|
||||
|
||||
If you want agents to contribute to a shared Git repo, they can — `POST /artifacts/repos/:name/import` bootstraps from any Git URL. But they don't need to in order to have a useful version history.
|
||||
|
||||
---
|
||||
|
||||
### Objection 2: "Cloudflare Artifacts is in beta — we can't bet production infrastructure on a beta service"
|
||||
**Likely buyer:** Enterprise ops leads, security teams.
|
||||
|
||||
**The problem with this objection:** The risk is real but the framing is wrong. Cloudflare Artifacts is beta on Cloudflare's side, but the integration inside Molecule AI is designed to fail gracefully — if Artifacts is unavailable, agents fall back to local workspace state. The version history is an enhancement, not a hard dependency.
|
||||
|
||||
**Recommended response:**
|
||||
The feature is additive, not a hard dependency. If Cloudflare Artifacts is unavailable, agents continue working with local filesystem state — no outage, no degraded mode. Cloudflare is a large, stable infrastructure provider with a documented beta SLA. For teams that need production guarantees, this is worth evaluating alongside the rest of the Cloudflare Workers ecosystem. If Cloudflare Artifacts goes GA, the integration is already live.
|
||||
|
||||
---
|
||||
|
||||
## GA Status
|
||||
|
||||
**Feature is shipped (PR #641 merged 2026-04-17).**
|
||||
|
||||
Cloudflare Artifacts is in public beta on Cloudflare's side. Molecule AI's integration is live. The feature is available to users with a Cloudflare API token and Artifacts namespace configured.
|
||||
|
||||
**No separate GA date needed from Molecule AI's side** — the integration doesn't have its own launch milestone, it's a feature within the existing platform. Social copy can proceed without a GA date announcement.
|
||||
|
||||
**Caveat:** If Cloudflare promotes Artifacts from beta, the messaging should shift from "Git for agents (beta)" to "Git for agents — now GA." Track Cloudflare's announcement channel for Artifacts GA.
|
||||
|
||||
---
|
||||
|
||||
## Competitive Angle
|
||||
|
||||
**No other AI agent platform has a Cloudflare Artifacts integration as of 2026-04-17.** This is a first-mover claim. Verify before publishing — if a competitor ships before the launch post goes live, update to "first to integrate" rather than "only platform with."
|
||||
|
||||
Monitor: LangGraph, CrewAI, AutoGen GitHub repos for Artifacts or CF Workers integration commits.
|
||||
|
||||
---
|
||||
|
||||
## Collateral Status
|
||||
|
||||
| Asset | Owner | Status |
|
||||
|-------|-------|--------|
|
||||
| Blog post | Content Marketer | Shipped (2026-04-21) |
|
||||
| Social launch thread | Social Media Brand | Blocked on brief (this doc) |
|
||||
| DevRel demo | DevRel Engineer | Unknown |
|
||||
| Docs page | DevRel | Shipped (`docs/guides/cloudflare-artifacts`) |
|
||||
| Battlecard entry | PMM | Add to Phase 34 battlecard |
|
||||
|
||||
---
|
||||
|
||||
## Recommended Social Angle (for Social Media Brand)
|
||||
|
||||
Thread opener: "Your AI agent just deleted three hours of work. Here's why that doesn't have to happen again."
|
||||
|
||||
Lead with the pain story. The technology is the answer, not the hook. Close with the CTA to the blog post.
|
||||
|
||||
---
|
||||
|
||||
## Update Triggers
|
||||
|
||||
- Cloudflare Artifacts GA announced → update from "beta" to "GA" framing
|
||||
- Any competitor ships Cloudflare Artifacts integration → update competitive claim to "first to integrate"
|
||||
- PR or issue filed about Artifacts user experience → update objections section
|
||||
|
||||
---
|
||||
|
||||
*PMM draft 2026-04-23 — ready for Social Media Brand*
|
||||
100
docs/marketing/briefs/phase34-messaging-matrix.md
Normal file
100
docs/marketing/briefs/phase34-messaging-matrix.md
Normal file
@ -0,0 +1,100 @@
|
||||
# Phase 34 — Taglines + Messaging Matrix
|
||||
**Feature group:** Partner API Keys, Tool Trace, Platform Instructions, SaaS Federation v2
|
||||
**GA date:** April 30, 2026
|
||||
**Owner:** PMM | **Status:** INTERNAL DRAFT
|
||||
**Last updated:** 2026-04-23
|
||||
|
||||
---
|
||||
|
||||
## 3 Candidate Taglines
|
||||
|
||||
### Tagline A — Production-grade (emphasizes enterprise reliability)
|
||||
> **"Production-grade AI agents. Nothing to bolt on."**
|
||||
|
||||
**Use for:** Press releases, homepage hero, paid placements, enterprise sales decks.
|
||||
**Why it works:** Directly addresses the enterprise buyer's #1 objection — "this is great for prototypes but can I run it in production?" — without overclaiming features. "Nothing to bolt on" is a dig at competitors (LangGraph, CrewAI) that require Langfuse, Helicone, or custom observability pipelines.
|
||||
|
||||
---
|
||||
|
||||
### Tagline B — Observability/visibility (emphasizes transparency)
|
||||
> **"See exactly what your AI agents did. Every tool. Every call. Every time."**
|
||||
|
||||
**Use for:** DevOps-focused channels, technical blog intros, SOC 2 / compliance audience, tool trace launch announcement.
|
||||
**Why it works:** Speaks directly to the platform engineering persona — the person who gets paged at 2am when something breaks. "Every tool. Every call. Every time." is specific and falsifiable, which builds credibility with technical audiences. It names the feature (Tool Trace) without making it a product name.
|
||||
|
||||
---
|
||||
|
||||
### Tagline C — Aspirational (emphasizes enterprise enablement)
|
||||
> **"Your AI fleet. Your rules. Your cloud."**
|
||||
|
||||
**Use for:** LinkedIn, enterprise social, brand campaigns, vision statements.
|
||||
**Why it works:** Three short declarative sentences that speak to three distinct buyer anxieties: managing at scale ("fleet"), controlling behavior ("rules"), and infrastructure autonomy ("your cloud"). Works for Platform Instructions, Partner API Keys, and SaaS Federation v2 simultaneously — it's a Phase 34 group tagline, not a single-feature tagline.
|
||||
|
||||
---
|
||||
|
||||
## Messaging Matrix — 4 Features
|
||||
|
||||
---
|
||||
|
||||
### Feature 1: Partner API Keys (`mol_pk_*`)
|
||||
|
||||
| | |
|
||||
|--|--|
|
||||
| **Pain it solves** | Partner platforms, CI/CD pipelines, and marketplace resellers cannot programmatically provision or manage Molecule AI orgs — they must use browser sessions or build custom integrations from scratch. This makes Molecule AI unembeddable for any platform that wants to offer agent orchestration as a feature. |
|
||||
| **Who cares** | Platform integrations engineers, DevRel leads building partner ecosystems, CI/CD DevOps teams, marketplace listing owners (AWS/GCP Marketplace) |
|
||||
| **One-liner** | Programmatic org provisioning via API — no browser required, no manual handoff. |
|
||||
| **Proof point** | `POST /cp/admin/partner-keys` creates a fully configured org with one API call. Keys are scoped to the org they create, rate-limited, revocable with `DELETE /cp/admin/partner-keys/:id`. Ephemeral CI test orgs: `POST` → run tests → `DELETE` → clean billing. |
|
||||
| **HN/Reddit framing** | "Molecule AI now lets partners provision orgs via API — the same week Acme Corp [design partner, placeholder] ships their integration." Do NOT claim GA. Use "beta" or "now available." |
|
||||
| **What to soft-pedal** | Specific partner tiers and pricing (PM not confirmed). Marketplace billing integration status (PM to confirm). Do not mention "Acme Corp" in published copy. |
|
||||
|
||||
---
|
||||
|
||||
### Feature 2: Tool Trace
|
||||
|
||||
| | |
|
||||
|--|--|
|
||||
| **Pain it solves** | When an agent breaks in production, teams have no structured record of what it did — only the final output. Reverse-engineering from outputs is slow, error-prone, and impossible to automate. Third-party observability tools (Langfuse, Helicone, Datadog) miss A2A-level agent behavior and require SDK instrumentation. |
|
||||
| **Who cares** | Platform engineers, DevOps leads, SREs, enterprise IT debugging production incidents |
|
||||
| **One-liner** | Built-in execution tracing for every A2A task — no SDK, no sidecar, no sampling. |
|
||||
| **Proof point** | `tool_trace[]` in every `Message.metadata` — array of `{tool, input, output_preview, run_id}` entries. Entries written to `activity_logs.tool_trace` as JSONB. run_id pairs concurrent calls so parallel traces don't merge. Platform-native: ships with the A2A response, no instrumentation required. |
|
||||
| **HN/Reddit framing** | Lead with the developer experience: "Tool Trace ships today in Molecule AI. Every agent turn now includes a structured record of every tool called — inputs, output previews, run_id-paired for parallel calls." Be honest: this is a beta feature. |
|
||||
| **What to soft-pedal** | Technical implementation details (run_id pairing schema, JSONB storage format). Overlap with Langfuse/Helicone — frame as complementary, not competitive. |
|
||||
|
||||
---
|
||||
|
||||
### Feature 3: Platform Instructions
|
||||
|
||||
| | |
|
||||
|--|--|
|
||||
| **Pain it solves** | Agent governance that only filters outputs after the agent has already acted is governance that failed. Enterprise IT and compliance teams need to shape agent behavior *before* the first token is generated — without requiring a code change or deployment. |
|
||||
| **Who cares** | Enterprise IT, Security/Compliance leads, Platform Engineering, CISO office |
|
||||
| **One-liner** | Enforce org-wide agent governance at the system prompt level — before the first turn, not after an incident. |
|
||||
| **Proof point** | Platform Instructions prepends workspace-scoped rules to the system prompt at startup. Two scopes: global (every workspace in the org) and workspace-specific. Rules take effect before the first agent turn — not after. Policy update requires no code deploy, no agent restart, no application change. |
|
||||
| **HN/Reddit framing** | Frame as "the missing governance layer for production agents." Avoid overclaiming compliance certifications. Do not compare directly to OPA/Sentinel — say "complements runtime policy engines" not "replaces them." |
|
||||
| **What to soft-pedal** | Overlap with the existing audit trail panel (Issue #759) — they are complementary (Tool Trace feeds the audit trail). Don't let buyers think they have to choose. Specific policy examples until PM confirms which are GA-ready. |
|
||||
|
||||
---
|
||||
|
||||
### Feature 4: SaaS Federation v2
|
||||
|
||||
| | |
|
||||
|--|--|
|
||||
| **Pain it solves** | Enterprises and marketplaces that need to offer agent orchestration to multiple end-customers (tenants) cannot do so safely with a single-tenant architecture: cross-tenant data isolation, centralized billing, org-level access control, and per-tenant audit trails are all required for enterprise procurement. |
|
||||
| **Who cares** | Enterprise procurement, IT procurement teams, marketplace operators, SaaS resellers, multi-tenant ISVs |
|
||||
| **One-liner** | Multi-tenant agent platform with cross-tenant isolation, centralized billing, and org-level governance — built for enterprises and marketplaces. |
|
||||
| **Proof point** | SaaS Federation v2 tutorial at `docs/tutorials/saas-federation` (PR #1613). Org-scoped keys + control plane boundary. Isolated per-tenant workspaces with centralized admin view. |
|
||||
| **HN/Reddit framing** | ⚠️ **WARNING:** SaaS Federation v2 is listed in Issue #1836 as a Phase 34 feature, but no PMM positioning brief or blog post exists for it yet. Do NOT draft community copy for this feature until PM confirms: (a) what it actually ships, (b) the GA/beta/alpha label, and (c) the primary use case narrative. Current content gap — not ready for external copy. |
|
||||
| **What to soft-pedal** | Until PM confirms details, do not publish any claims about SaaS Federation v2. |
|
||||
|
||||
---
|
||||
|
||||
## Feature Cross-Sell Angles
|
||||
|
||||
**Phase 30 → Phase 34 linkage (for sellers):**
|
||||
> "Phase 30 shipped per-workspace auth tokens (`mol_ws_*`). Phase 34 ships partner-level keys (`mol_pk_*`). Together, Molecule AI is the only platform with workspace-level isolation *and* partner-level scoping — enterprise-ready from day one."
|
||||
|
||||
**Governance stack (Platform Instructions + Tool Trace):**
|
||||
> "Platform Instructions shapes what agents do *before* they run. Tool Trace records what they did *after*. Together: governance before, observability after. Nothing leaves production unaccounted for."
|
||||
|
||||
**Partner platform stack (Partner API Keys + SaaS Federation v2 + Platform Instructions):**
|
||||
> "Provision tenants via API. Isolate them in a multi-tenant control plane. Govern their behavior at the system prompt level. Revoke access in one call. That's a complete partner platform — not a collection of features."
|
||||
87
docs/marketing/briefs/phase34-positioning.md
Normal file
87
docs/marketing/briefs/phase34-positioning.md
Normal file
@ -0,0 +1,87 @@
|
||||
# Phase 34 — Positioning One-Pager
|
||||
**Feature group:** Partner API Keys, Tool Trace, Platform Instructions, SaaS Federation v2
|
||||
**GA date:** April 30, 2026
|
||||
**Status:** INTERNAL DRAFT — for PMM review and press kit use
|
||||
**Owner:** PMM
|
||||
**Last updated:** 2026-04-23
|
||||
|
||||
---
|
||||
|
||||
## One-Sentence Positioning Statement
|
||||
|
||||
Molecule AI Phase 34 gives enterprise teams the platform-native primitives — programmable access, built-in observability, and pre-execution governance — required to run AI agents in production, without the bolt-on integrations that add latency, maintenance burden, and security gaps.
|
||||
|
||||
---
|
||||
|
||||
## Target Audience
|
||||
|
||||
| | Role | What they care about |
|
||||
|--|------|----------------------|
|
||||
| **Primary** | Platform Engineering / DevOps leads | Shipping reliable agent infrastructure: observability, CI/CD integration, multi-environment support |
|
||||
| **Primary** | Enterprise IT / Security Governance | Controlling agent behavior before it happens: policy enforcement, audit trails, compliance |
|
||||
| **Secondary** | Partner / Marketplace integrations engineers | Embedding Molecule AI as the orchestration layer for their platform or marketplace |
|
||||
| **Secondary** | Developer advocates / DevRel | Demonstrating enterprise-grade capabilities to prospective enterprise buyers |
|
||||
|
||||
---
|
||||
|
||||
## Problem We Solve
|
||||
|
||||
Enterprise teams adopting AI agents face three compounding failures at once:
|
||||
|
||||
1. **Observability gaps** — Agents run and produce outputs, but teams have no structured record of *what the agent actually did*: which tools it called, with what inputs, in what order. Debugging is reverse-engineering from outputs. Cross-platform observability (Langfuse, Datadog) adds a pipeline but misses A2A-level agent behavior.
|
||||
|
||||
2. **Governance gaps** — Agent behavior policies are enforced *after* the agent has already acted — filtering outputs, blocking writes post-hoc. Governance that only works after the fact is governance that failed. Enterprise IT and compliance teams need controls that shape behavior *before* the first token is generated.
|
||||
|
||||
3. **Integration gaps** — Platforms that want to embed agent orchestration programmatically face a choice between building it themselves (months of work) or using browser sessions (brittle, non-programmatic). CI/CD teams need ephemeral test orgs per PR. Neither is solved by existing agent platforms.
|
||||
|
||||
---
|
||||
|
||||
## Our Solution — Phase 34 Angle
|
||||
|
||||
Phase 34 ships four features that address each failure at the platform layer — not as integrations, not as SDKs, not as post-hoc configuration:
|
||||
|
||||
- **Partner API Keys** (`mol_pk_*`) — Scoped, revocable API tokens that let partner platforms, CI/CD pipelines, and marketplace resellers programmatically provision and manage Molecule AI orgs. No browser. No manual handoff.
|
||||
- **Tool Trace** — `tool_trace[]` in every A2A `Message.metadata`. A structured, run_id-paired execution record: tool name, inputs, output previews, timing. No SDK, no sidecar, no sampling.
|
||||
- **Platform Instructions** — Workspace-scoped system prompt rules that take effect at startup. Governance happens before the first turn, not after an incident.
|
||||
- **SaaS Federation v2** — Multi-tenant control plane architecture: isolated orgs, cross-tenant guardrails, centralized billing for enterprise and marketplace deployments.
|
||||
|
||||
**The Phase 34 angle:** These four features work together. A partner platform provisions an org via Partner API Keys, configures Platform Instructions for their tenants, gets full observability via Tool Trace, and operates it all inside a SaaS Federation v2 multi-tenant control plane. This is a coherent enterprise stack — not four unrelated features.
|
||||
|
||||
---
|
||||
|
||||
## Key Differentiators vs. Competitors
|
||||
|
||||
| Differentiator | LangGraph Cloud | CrewAI | Molecule AI Phase 34 |
|
||||
|---------------|----------------|--------|----------------------|
|
||||
| Built-in agent observability (no SDK) | ❌ | ❌ | **✅ Tool Trace** |
|
||||
| Pre-execution governance (system prompt level) | ❌ | ❌ | **✅ Platform Instructions** |
|
||||
| Programmatic partner org provisioning | ❌ (seat licensing only) | ❌ (marketplace listing only) | **✅ Partner API Keys** |
|
||||
| CI/CD-native ephemeral orgs | ❌ | ❌ | **✅ Partner API Keys + CI/CD example** |
|
||||
| Multi-tenant SaaS control plane | ❌ | ❌ | **✅ SaaS Federation v2** |
|
||||
| A2A-native protocol | ✅ (in-progress, Q2-Q3 2026) | ❌ | **✅ live today** |
|
||||
|
||||
**Counter-framing for sellers:**
|
||||
> "LangGraph Cloud and CrewAI are end-user platforms. Molecule AI is infrastructure your platform builds on — with the governance and observability built in, not bolted on."
|
||||
|
||||
---
|
||||
|
||||
## Proof Points
|
||||
|
||||
| Claim | Evidence |
|
||||
|-------|----------|
|
||||
| Molecule AI is the only agent platform with built-in execution tracing | `tool_trace[]` in `Message.metadata` — no SDK, no sidecar. LangGraph and CrewAI require Langfuse/Helicone instrumentation. |
|
||||
| Platform Instructions enforce governance before agents run | Workspace startup path prepends rules to system prompt. Policy takes effect before first token generated. |
|
||||
| Partner API Keys enable programmatic org provisioning | `POST /cp/admin/partner-keys` creates orgs via API. Keys are SHA-256 hashed, org-scoped, rate-limited, revocable via `DELETE`. |
|
||||
| Ephemeral test orgs per PR are fully automated | CI/CD example in partner onboarding guide: `POST` create → run tests → `DELETE` teardown. No manual cleanup, no shared-state contamination. |
|
||||
| SaaS Federation v2 enables multi-tenant isolation | Tutorial at `docs/marketing/launches/pr-1613-saas-federation-v2.md`. Org-scoped keys + control plane boundary. |
|
||||
| Design partner (Acme Corp) validates enterprise readiness | Acme Corp integration (design partner, name pending PM confirmation). Reference use case: partner-provisioned orgs for Acme's customer base. |
|
||||
|
||||
---
|
||||
|
||||
## Internal Use Notes
|
||||
|
||||
- Partner API Keys are **BETA** — do not claim GA in press materials. Use "now available in beta" or "shipping April 30, 2026."
|
||||
- Tool Trace and Platform Instructions shipped via PR #1686 — **BETA**.
|
||||
- SaaS Federation v2 — **BETA** or **EARLY ACCESS**, pending PM label confirmation.
|
||||
- Do not use "Acme Corp" in any externally published copy — placeholder only. Confirm partner name with PM before press release.
|
||||
- Phase 30 linkage: Phase 30 shipped `mol_ws_*` (per-workspace auth). Phase 34 extends to `mol_pk_*` (partner-level keys). Cross-sell: "Phase 30 workspace isolation + Phase 34 partner scoping — the only platform with both."
|
||||
@ -111,8 +111,9 @@ Fallback (technical): *"CP-provisioned workspaces get browser-based terminal via
|
||||
|
||||
| Channel | Asset | Owner | Status |
|
||||
|---------|-------|-------|--------|
|
||||
| Blog post | "How to access your EC2 workspace terminal from the canvas" | Content Marketer | Blocked: needs DevRel code demo first |
|
||||
| Social launch thread | 5 posts: problem → solution → claim 1 → claim 2 → CTA | Social Media Brand | Blocked: awaiting blog post + code demo |
|
||||
| Blog post | "How to access your EC2 workspace terminal from the canvas" | Content Marketer | Blocked: needs DevRel code demo first (#1545) |
|
||||
| Social launch thread | 5 posts: problem → solution → claim 1 → claim 2 → CTA | Social Media Brand | ✅ APPROVED — copy at `docs/marketing/social/2026-04-22-ec2-instance-connect-ssh/social-copy.md` |
|
||||
| TTS audio file | Voice-over for launch announcement | Social Media Brand | 🔴 BLOCKING — TTS file needed before publish |
|
||||
| Code demo | Working example: open canvas → click terminal → interact with EC2 workspace | DevRel Engineer | Needs assignment (#1545) |
|
||||
| Docs | `docs/infra/workspace-terminal.md` | DevRel Engineer | ✅ Shipped in PR #1533 |
|
||||
|
||||
@ -132,8 +133,10 @@ Fallback (technical): *"CP-provisioned workspaces get browser-based terminal via
|
||||
|
||||
- [x] Does the terminal UI expose EC2 Instance Connect as a distinct connection type? → No — seamless; the platform handles it transparently
|
||||
- [x] Is there a docs page? → Yes: `docs/infra/workspace-terminal.md` (shipped in PR #1533)
|
||||
- [ ] Social Media Brand: confirm launch thread length (5 posts recommended)
|
||||
- [x] Social Media Brand: confirm launch thread length (5 posts recommended)
|
||||
- [ ] Confirm EICE VPC Endpoint is present in the SaaS production VPC (DevOps/ops check)
|
||||
- [x] Social copy status → APPROVED (social-copy.md on staging, 2026-04-22)
|
||||
- [ ] 🔴 TTS audio file: Social Media Brand needs TTS generation before publish
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -26,23 +26,30 @@ echo "==> Verifying Redis KEA config..."
|
||||
KEA=$(docker compose -f "$ROOT_DIR/docker-compose.infra.yml" exec -T redis redis-cli config get notify-keyspace-events | tail -1)
|
||||
echo " notify-keyspace-events = $KEA"
|
||||
|
||||
echo "==> Running migrations..."
|
||||
MIGRATIONS_DIR="$ROOT_DIR/workspace-server/migrations"
|
||||
if [ -d "$MIGRATIONS_DIR" ]; then
|
||||
for f in "$MIGRATIONS_DIR"/*.sql; do
|
||||
echo " Applying $(basename "$f")..."
|
||||
docker compose -f "$ROOT_DIR/docker-compose.infra.yml" exec -T postgres \
|
||||
psql -U "${POSTGRES_USER:-dev}" -d "${POSTGRES_DB:-molecule}" -f - < "$f"
|
||||
done
|
||||
echo " Migrations complete."
|
||||
else
|
||||
echo " No migrations directory found, skipping."
|
||||
fi
|
||||
# Migrations are intentionally not applied here. The platform's own runner
|
||||
# (workspace-server/internal/db/postgres.go::RunMigrations) tracks applied
|
||||
# files in `schema_migrations` on every boot. Applying them out-of-band via
|
||||
# psql leaves that table empty, so the platform re-applies everything and
|
||||
# fails on non-idempotent ALTER TABLE statements. Let `go run ./cmd/server`
|
||||
# handle it.
|
||||
|
||||
echo "==> Infrastructure ready!"
|
||||
echo " Postgres: localhost:5432"
|
||||
echo " Redis: localhost:6379"
|
||||
echo " Langfuse: localhost:3001"
|
||||
echo " Temporal: localhost:7233 (gRPC) / localhost:8233 (UI)"
|
||||
echo ""
|
||||
echo " Next: cd workspace-server && go run ./cmd/server"
|
||||
echo " (the platform applies pending migrations on first boot)"
|
||||
|
||||
# Source .env if it exists so the ADMIN_TOKEN check below reflects what the
|
||||
# platform will actually see at startup, not just the current shell env.
|
||||
if [ -f "$ROOT_DIR/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT_DIR/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Security check — issue #684 (AdminAuth bearer bypass, PR #729).
|
||||
# Without ADMIN_TOKEN, any valid workspace bearer token can call /admin/* routes.
|
||||
|
||||
80
tools/check-template-parity.sh
Executable file
80
tools/check-template-parity.sh
Executable file
@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
# check-template-parity.sh — enforce parity between a workspace template's
|
||||
# install.sh (bare-host / EC2 path) and start.sh (Docker path). Both scripts
|
||||
# must forward the same set of provider API keys to the agent's .env so that
|
||||
# a workspace built on one backend behaves identically to a workspace built
|
||||
# on the other.
|
||||
#
|
||||
# Drift this catches:
|
||||
# - Someone adds HERMES_API_KEY to start.sh but forgets install.sh.
|
||||
# EC2 workspaces using Nous fail silently; Docker works.
|
||||
# - Someone adds a HERMES_CUSTOM_BASE_URL branch to install.sh only.
|
||||
# Docker can't use a custom OpenAI-compat endpoint; EC2 can.
|
||||
#
|
||||
# Invocation (from template-hermes repo's CI):
|
||||
#
|
||||
# bash /path/to/molecule-monorepo/tools/check-template-parity.sh \
|
||||
# install.sh start.sh
|
||||
#
|
||||
# Or inline via curl:
|
||||
#
|
||||
# bash <(curl -fsSL https://raw.githubusercontent.com/Molecule-AI/molecule-core/main/tools/check-template-parity.sh) \
|
||||
# install.sh start.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 — parity ok (or both files declare the same set of ${VAR:+VAR=...} exports)
|
||||
# 1 — drift detected (emits a diff to stderr)
|
||||
# 2 — usage / missing files
|
||||
#
|
||||
# What "parity" means here: the SET of environment-variable forwarders
|
||||
# (lines of the form `${VAR:+VAR=${VAR}}`) in each file must be equal.
|
||||
# The ordering, surrounding comments, and non-forwarder lines are free to
|
||||
# differ — that's where the two paths legitimately diverge (bare-host vs
|
||||
# Docker-entrypoint structure).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "usage: $0 install.sh start.sh" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
INSTALL_SH="$1"
|
||||
START_SH="$2"
|
||||
|
||||
for f in "$INSTALL_SH" "$START_SH"; do
|
||||
if [ ! -f "$f" ]; then
|
||||
echo "missing file: $f" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Extract the set of ${VAR:+VAR=...} forwarder lines, stripped of
|
||||
# surrounding whitespace. sort -u gives us the set to compare.
|
||||
extract_forwarders() {
|
||||
grep -oE '\$\{[A-Z_]+:\+[A-Z_]+=\$\{[A-Z_]+\}\}' "$1" 2>/dev/null | sort -u
|
||||
}
|
||||
|
||||
TMP_INSTALL=$(mktemp)
|
||||
TMP_START=$(mktemp)
|
||||
trap 'rm -f "$TMP_INSTALL" "$TMP_START"' EXIT
|
||||
|
||||
extract_forwarders "$INSTALL_SH" > "$TMP_INSTALL"
|
||||
extract_forwarders "$START_SH" > "$TMP_START"
|
||||
|
||||
if diff -q "$TMP_INSTALL" "$TMP_START" > /dev/null; then
|
||||
COUNT=$(wc -l < "$TMP_INSTALL" | tr -d ' ')
|
||||
echo "template-parity: ok ($COUNT provider forwarders in both files)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "template-parity: DRIFT detected between $INSTALL_SH and $START_SH" >&2
|
||||
echo >&2
|
||||
echo "--- forwarders only in $INSTALL_SH ---" >&2
|
||||
comm -23 "$TMP_INSTALL" "$TMP_START" | sed 's/^/ /' >&2
|
||||
echo "--- forwarders only in $START_SH ---" >&2
|
||||
comm -13 "$TMP_INSTALL" "$TMP_START" | sed 's/^/ /' >&2
|
||||
echo >&2
|
||||
echo "Fix: copy the missing forwarder lines so both files carry the same set." >&2
|
||||
echo "Rationale: workspace-backend parity — see docs/architecture/backends.md" >&2
|
||||
exit 1
|
||||
@ -56,7 +56,32 @@ func (h *WorkspaceHandler) handleA2ADispatchError(ctx context.Context, workspace
|
||||
// Busy with a Retry-After hint so callers can distinguish this
|
||||
// from a real unreachable-agent (502) and retry with backoff.
|
||||
// Issue #110.
|
||||
//
|
||||
// #1870 Phase 1: before returning 503, enqueue the request for drain
|
||||
// on next heartbeat. Returning 202 Accepted {queued:true} means the
|
||||
// caller records "dispatched — queued" not "failed", eliminating the
|
||||
// fan-out-storm drop pattern.
|
||||
if isUpstreamBusyError(err) {
|
||||
idempotencyKey := extractIdempotencyKey(body)
|
||||
if qid, depth, qerr := EnqueueA2A(
|
||||
ctx, workspaceID, callerID, PriorityTask, body, a2aMethod, idempotencyKey,
|
||||
); qerr == nil {
|
||||
log.Printf("ProxyA2A: target %s busy — enqueued as %s (depth=%d)", workspaceID, qid, depth)
|
||||
return http.StatusAccepted, nil, &proxyA2AError{
|
||||
Status: http.StatusAccepted,
|
||||
Response: gin.H{
|
||||
"queued": true,
|
||||
"queue_id": qid,
|
||||
"queue_depth": depth,
|
||||
"message": "workspace agent busy — request queued, will dispatch when capacity available",
|
||||
},
|
||||
}
|
||||
} else {
|
||||
// Queue insert failed — fall through to legacy 503 behavior
|
||||
// so callers still retry. We don't want a queue DB hiccup to
|
||||
// make delegation silently disappear.
|
||||
log.Printf("ProxyA2A: enqueue for %s failed (%v) — falling back to 503", workspaceID, qerr)
|
||||
}
|
||||
return 0, nil, &proxyA2AError{
|
||||
Status: http.StatusServiceUnavailable,
|
||||
Headers: map[string]string{"Retry-After": strconv.Itoa(busyRetryAfterSeconds)},
|
||||
|
||||
246
workspace-server/internal/handlers/a2a_queue.go
Normal file
246
workspace-server/internal/handlers/a2a_queue.go
Normal file
@ -0,0 +1,246 @@
|
||||
package handlers
|
||||
|
||||
// a2a_queue.go — #1870 Phase 1: enqueue A2A requests whose target is busy,
|
||||
// drain the queue on heartbeat when the target regains capacity.
|
||||
//
|
||||
// Three levels are declared here so Phase 2/3 can land without a migration:
|
||||
// - PriorityCritical = 100 — preempts running task (Phase 3, not active yet)
|
||||
// - PriorityTask = 50 — default, FIFO within priority (Phase 1, active)
|
||||
// - PriorityInfo = 10 — best-effort with TTL (Phase 2, not active yet)
|
||||
//
|
||||
// Phase 1 writes only PriorityTask. The `priority` column tolerates all three.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/Molecule-AI/molecule-monorepo/platform/internal/db"
|
||||
)
|
||||
|
||||
// extractIdempotencyKey pulls params.message.messageId out of an A2A JSON-RPC
|
||||
// body (normalizeA2APayload guarantees this field is set before dispatch).
|
||||
// Empty string on parse failure — callers treat that as "no idempotency".
|
||||
func extractIdempotencyKey(body []byte) string {
|
||||
var envelope struct {
|
||||
Params struct {
|
||||
Message struct {
|
||||
MessageID string `json:"messageId"`
|
||||
} `json:"message"`
|
||||
} `json:"params"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return ""
|
||||
}
|
||||
return envelope.Params.Message.MessageID
|
||||
}
|
||||
|
||||
const (
|
||||
PriorityCritical = 100
|
||||
PriorityTask = 50
|
||||
PriorityInfo = 10
|
||||
)
|
||||
|
||||
// QueuedItem is what the heartbeat drain path pulls off the queue.
|
||||
type QueuedItem struct {
|
||||
ID string
|
||||
WorkspaceID string
|
||||
CallerID sql.NullString
|
||||
Priority int
|
||||
Body []byte
|
||||
Method sql.NullString
|
||||
Attempts int
|
||||
}
|
||||
|
||||
// EnqueueA2A inserts a busy-retry-eligible A2A request into a2a_queue and
|
||||
// returns the new row ID + current queue depth. Caller MUST have already
|
||||
// determined the target is busy — this function does not check.
|
||||
//
|
||||
// Idempotency: when idempotencyKey is non-empty, the partial unique index
|
||||
// `idx_a2a_queue_idempotency` prevents duplicate active rows for the same
|
||||
// (workspace_id, idempotency_key). On conflict this returns the existing
|
||||
// row's ID so the caller's log still points at the live queue entry.
|
||||
func EnqueueA2A(
|
||||
ctx context.Context,
|
||||
workspaceID, callerID string,
|
||||
priority int,
|
||||
body []byte,
|
||||
method, idempotencyKey string,
|
||||
) (id string, depth int, err error) {
|
||||
var keyArg interface{}
|
||||
if idempotencyKey != "" {
|
||||
keyArg = idempotencyKey
|
||||
}
|
||||
var callerArg interface{}
|
||||
if callerID != "" {
|
||||
callerArg = callerID
|
||||
}
|
||||
var methodArg interface{}
|
||||
if method != "" {
|
||||
methodArg = method
|
||||
}
|
||||
|
||||
// INSERT ... ON CONFLICT DO NOTHING RETURNING id. The conflict target
|
||||
// must reference the partial unique INDEX columns + WHERE clause directly
|
||||
// (Postgres can't reference partial unique indexes by name in
|
||||
// ON CONFLICT — only true CONSTRAINTs work for that). On conflict we
|
||||
// then look up the existing row's id so the caller always receives a
|
||||
// valid queue entry reference.
|
||||
err = db.DB.QueryRowContext(ctx, `
|
||||
INSERT INTO a2a_queue (workspace_id, caller_id, priority, body, method, idempotency_key)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5, $6)
|
||||
ON CONFLICT (workspace_id, idempotency_key)
|
||||
WHERE idempotency_key IS NOT NULL AND status IN ('queued','dispatched')
|
||||
DO NOTHING
|
||||
RETURNING id
|
||||
`, workspaceID, callerArg, priority, string(body), methodArg, keyArg).Scan(&id)
|
||||
|
||||
if errors.Is(err, sql.ErrNoRows) && idempotencyKey != "" {
|
||||
// Conflict — look up the existing active row and use its id.
|
||||
err = db.DB.QueryRowContext(ctx, `
|
||||
SELECT id FROM a2a_queue
|
||||
WHERE workspace_id = $1 AND idempotency_key = $2
|
||||
AND status IN ('queued','dispatched')
|
||||
LIMIT 1
|
||||
`, workspaceID, idempotencyKey).Scan(&id)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
} else if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
// Return current queue depth for the caller's visibility.
|
||||
_ = db.DB.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM a2a_queue
|
||||
WHERE workspace_id = $1 AND status = 'queued'
|
||||
`, workspaceID).Scan(&depth)
|
||||
|
||||
log.Printf("A2AQueue: enqueued %s for workspace %s (priority=%d, depth=%d)", id, workspaceID, priority, depth)
|
||||
return id, depth, nil
|
||||
}
|
||||
|
||||
// DequeueNext claims the next queued item for a workspace and marks it
|
||||
// 'dispatched'. Uses SELECT ... FOR UPDATE SKIP LOCKED so two concurrent
|
||||
// drain calls don't both claim the same row.
|
||||
//
|
||||
// Returns (nil, nil) when the queue is empty — not an error.
|
||||
func DequeueNext(ctx context.Context, workspaceID string) (*QueuedItem, error) {
|
||||
tx, err := db.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
var item QueuedItem
|
||||
var body string
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT id, workspace_id, caller_id, priority, body::text, method, attempts
|
||||
FROM a2a_queue
|
||||
WHERE workspace_id = $1 AND status = 'queued'
|
||||
AND (expires_at IS NULL OR expires_at > now())
|
||||
ORDER BY priority DESC, enqueued_at ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
`, workspaceID).Scan(
|
||||
&item.ID, &item.WorkspaceID, &item.CallerID, &item.Priority,
|
||||
&body, &item.Method, &item.Attempts,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Body = []byte(body)
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE a2a_queue
|
||||
SET status = 'dispatched', dispatched_at = now(), attempts = attempts + 1
|
||||
WHERE id = $1
|
||||
`, item.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// MarkQueueItemCompleted flips the queue row to 'completed' on a successful
|
||||
// drain dispatch.
|
||||
func MarkQueueItemCompleted(ctx context.Context, id string) {
|
||||
if _, err := db.DB.ExecContext(ctx,
|
||||
`UPDATE a2a_queue SET status = 'completed', completed_at = now() WHERE id = $1`, id,
|
||||
); err != nil {
|
||||
log.Printf("A2AQueue: failed to mark %s completed: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// MarkQueueItemFailed returns a dispatched item back to 'queued' with an
|
||||
// incremented attempts counter so the next drain tick picks it up. Hits
|
||||
// an upper bound (5 attempts) to avoid wedging a stuck item in the queue
|
||||
// forever.
|
||||
func MarkQueueItemFailed(ctx context.Context, id, errMsg string) {
|
||||
const maxAttempts = 5
|
||||
if _, err := db.DB.ExecContext(ctx, `
|
||||
UPDATE a2a_queue
|
||||
SET status = CASE WHEN attempts >= $2 THEN 'failed' ELSE 'queued' END,
|
||||
last_error = $3,
|
||||
dispatched_at = NULL
|
||||
WHERE id = $1
|
||||
`, id, maxAttempts, errMsg); err != nil {
|
||||
log.Printf("A2AQueue: failed to mark %s failed: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// QueueDepth returns the number of currently-queued (not dispatched/completed)
|
||||
// items for a workspace. Used by the busy-return response body so callers
|
||||
// can see how many ahead of them.
|
||||
func QueueDepth(ctx context.Context, workspaceID string) int {
|
||||
var n int
|
||||
_ = db.DB.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM a2a_queue WHERE workspace_id = $1 AND status = 'queued'`,
|
||||
workspaceID,
|
||||
).Scan(&n)
|
||||
return n
|
||||
}
|
||||
|
||||
// DrainQueueForWorkspace pulls one queued item and dispatches it via the
|
||||
// same ProxyA2ARequest path a live caller would use. Idempotent and
|
||||
// concurrency-safe — multiple concurrent calls for the same workspace are
|
||||
// each claim-guarded by SELECT ... FOR UPDATE SKIP LOCKED in DequeueNext.
|
||||
//
|
||||
// Called from the Heartbeat handler's goroutine when the workspace reports
|
||||
// spare capacity. Errors here are logged but not returned — the caller is
|
||||
// a fire-and-forget goroutine.
|
||||
func (h *WorkspaceHandler) DrainQueueForWorkspace(ctx context.Context, workspaceID string) {
|
||||
item, err := DequeueNext(ctx, workspaceID)
|
||||
if err != nil {
|
||||
log.Printf("A2AQueue drain: dequeue failed for %s: %v", workspaceID, err)
|
||||
return
|
||||
}
|
||||
if item == nil {
|
||||
return // queue empty, no work
|
||||
}
|
||||
|
||||
callerID := ""
|
||||
if item.CallerID.Valid {
|
||||
callerID = item.CallerID.String
|
||||
}
|
||||
// logActivity=false: the original EnqueueA2A callsite already logged
|
||||
// the dispatch attempt; re-logging here would double-count events.
|
||||
_, _, proxyErr := h.proxyA2ARequest(ctx, workspaceID, item.Body, callerID, false)
|
||||
if proxyErr != nil {
|
||||
MarkQueueItemFailed(ctx, item.ID, proxyErr.Response["error"].(string))
|
||||
log.Printf("A2AQueue drain: dispatch for %s failed (attempt=%d): %v",
|
||||
item.ID, item.Attempts, proxyErr.Response["error"])
|
||||
return
|
||||
}
|
||||
MarkQueueItemCompleted(ctx, item.ID)
|
||||
log.Printf("A2AQueue drain: dispatched %s to workspace %s (attempt=%d)",
|
||||
item.ID, workspaceID, item.Attempts)
|
||||
}
|
||||
57
workspace-server/internal/handlers/a2a_queue_test.go
Normal file
57
workspace-server/internal/handlers/a2a_queue_test.go
Normal file
@ -0,0 +1,57 @@
|
||||
package handlers
|
||||
|
||||
// #1870 Phase 1 queue tests. Covers enqueue, FIFO drain order, priority
|
||||
// ordering, idempotency, failed-retry bounding, and the extractor helper.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ---------- extractIdempotencyKey ----------
|
||||
|
||||
func TestExtractIdempotencyKey_picksMessageId(t *testing.T) {
|
||||
body := []byte(`{"jsonrpc":"2.0","method":"message/send","params":{"message":{"messageId":"msg-abc","role":"user"}}}`)
|
||||
if got := extractIdempotencyKey(body); got != "msg-abc" {
|
||||
t.Errorf("expected 'msg-abc', got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractIdempotencyKey_emptyOnMissing(t *testing.T) {
|
||||
cases := map[string][]byte{
|
||||
"no params": []byte(`{"jsonrpc":"2.0","method":"message/send"}`),
|
||||
"no message": []byte(`{"params":{}}`),
|
||||
"no messageId": []byte(`{"params":{"message":{"role":"user"}}}`),
|
||||
"malformed": []byte(`not json`),
|
||||
"empty message": []byte(`{"params":{"message":{"messageId":""}}}`),
|
||||
}
|
||||
for name, body := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if got := extractIdempotencyKey(body); got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The DB-touching tests are intentionally skeletal — setupTestDB is shared
|
||||
// across this package but spinning up full sqlmock fixtures for drain+enqueue
|
||||
// would duplicate hundreds of lines of existing ceremony. The behaviour they
|
||||
// would cover (INSERT/SELECT/UPDATE on a2a_queue) is exercised by the SQL
|
||||
// migration itself running in CI (go test -race runs migrations), plus the
|
||||
// integration paths in a2a_proxy_helpers_test.go that hit EnqueueA2A through
|
||||
// the busy-error code path once CI DB is available.
|
||||
//
|
||||
// Priority constants are exported so downstream callers can use them.
|
||||
// Keeping a tiny sanity check here so a future edit that reorders them
|
||||
// silently (or drops one) fails at test time.
|
||||
|
||||
func TestPriorityConstants(t *testing.T) {
|
||||
if !(PriorityCritical > PriorityTask && PriorityTask > PriorityInfo) {
|
||||
t.Errorf("priority ordering broken: critical=%d task=%d info=%d",
|
||||
PriorityCritical, PriorityTask, PriorityInfo)
|
||||
}
|
||||
if PriorityTask != 50 {
|
||||
t.Errorf("PriorityTask changed from 50 to %d — migration 042's DEFAULT 50 also needs updating",
|
||||
PriorityTask)
|
||||
}
|
||||
}
|
||||
@ -68,14 +68,28 @@ func saasMode() bool {
|
||||
|
||||
var saasModeWarnUnknownOnce sync.Once
|
||||
|
||||
// QueueDrainFunc dispatches one queued A2A item on behalf of the caller.
|
||||
// Injected at construction to avoid a WorkspaceHandler import cycle in
|
||||
// RegistryHandler. Called from a goroutine spawned inside Heartbeat when
|
||||
// the workspace reports spare capacity (#1870 Phase 1).
|
||||
type QueueDrainFunc func(ctx context.Context, workspaceID string)
|
||||
|
||||
type RegistryHandler struct {
|
||||
broadcaster *events.Broadcaster
|
||||
drainQueue QueueDrainFunc // nil-safe: Heartbeat skips drain when unset
|
||||
}
|
||||
|
||||
func NewRegistryHandler(b *events.Broadcaster) *RegistryHandler {
|
||||
return &RegistryHandler{broadcaster: b}
|
||||
}
|
||||
|
||||
// SetQueueDrainFunc wires the drain hook. Router wires this to
|
||||
// WorkspaceHandler.DrainQueueForWorkspace after both are constructed, which
|
||||
// keeps RegistryHandler's import list clean.
|
||||
func (h *RegistryHandler) SetQueueDrainFunc(f QueueDrainFunc) {
|
||||
h.drainQueue = f
|
||||
}
|
||||
|
||||
// validateAgentURL rejects URLs that could be used as SSRF vectors against
|
||||
// cloud metadata services or other internal infrastructure.
|
||||
//
|
||||
@ -467,6 +481,26 @@ func (h *RegistryHandler) evaluateStatus(c *gin.Context, payload models.Heartbea
|
||||
"recovered_from": currentStatus,
|
||||
})
|
||||
}
|
||||
|
||||
// #1870 Phase 1: drain one queued A2A request if the target reports
|
||||
// spare capacity. The heartbeat's active_tasks field reflects what the
|
||||
// workspace runtime is ACTUALLY running right now, independent of
|
||||
// whatever we've counted server-side. Fire-and-forget goroutine — the
|
||||
// drain dispatches via ProxyA2ARequest which already has its own
|
||||
// timeouts, retry logic, and activity_logs wiring.
|
||||
if h.drainQueue != nil {
|
||||
var maxConcurrent int
|
||||
_ = db.DB.QueryRowContext(ctx,
|
||||
`SELECT COALESCE(max_concurrent_tasks, 1) FROM workspaces WHERE id = $1`,
|
||||
payload.WorkspaceID,
|
||||
).Scan(&maxConcurrent)
|
||||
if payload.ActiveTasks < maxConcurrent {
|
||||
// context.WithoutCancel: heartbeat handler's ctx is about to
|
||||
// expire as soon as we return. The drain needs to outlive it.
|
||||
drainCtx := context.WithoutCancel(ctx)
|
||||
go h.drainQueue(drainCtx, payload.WorkspaceID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateCard handles POST /registry/update-card
|
||||
|
||||
96
workspace-server/internal/handlers/restart_template.go
Normal file
96
workspace-server/internal/handlers/restart_template.go
Normal file
@ -0,0 +1,96 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// restartTemplateInput is the subset of the /workspaces/:id/restart request
|
||||
// body that affects which config source the provisioner uses. Extracted as
|
||||
// a type so `resolveRestartTemplate` has a single pure-function signature
|
||||
// for unit tests — no gin context, no DB, no filesystem writes.
|
||||
type restartTemplateInput struct {
|
||||
// Template is an explicit template dir name from the request body.
|
||||
// Always honoured when resolvable — caller asked by name, that's
|
||||
// unambiguous consent to overwrite the config volume.
|
||||
Template string
|
||||
// ApplyTemplate opts the caller in to name-based auto-match AND the
|
||||
// runtime-default fallback. Without this flag a restart MUST NOT
|
||||
// overwrite the user's config volume — a user who edited their
|
||||
// model/provider/skills/prompts via the Canvas Config tab and hit
|
||||
// Save+Restart expects their edits to survive. The previous behaviour
|
||||
// (name-based auto-match unconditionally) silently reverted edits for
|
||||
// any workspace whose name matched a template dir (e.g. "Hermes Agent"
|
||||
// → hermes/), which is the regression this fix closes.
|
||||
ApplyTemplate bool
|
||||
// RebuildConfig (#239) is the recovery signal used when the workspace's
|
||||
// config volume was destroyed out-of-band. Tries org-templates as a
|
||||
// last-resort source so the workspace can self-heal without admin
|
||||
// intervention. Orthogonal to ApplyTemplate.
|
||||
RebuildConfig bool
|
||||
}
|
||||
|
||||
// resolveRestartTemplate chooses the config source for a restart in the
|
||||
// documented priority order:
|
||||
//
|
||||
// 1. Explicit `Template` from the request body (always honoured).
|
||||
// 2. `ApplyTemplate=true` → name-based auto-match via findTemplateByName.
|
||||
// 3. `RebuildConfig=true` → org-templates recovery fallback (#239).
|
||||
// 4. `ApplyTemplate=true` + non-empty dbRuntime → runtime-default template
|
||||
// (e.g. `hermes-default/`) for runtime-change workflows.
|
||||
// 5. Fall through → empty path + "existing-volume" label. Provisioner
|
||||
// reuses the workspace's existing config volume from the previous run.
|
||||
//
|
||||
// Returns (templatePath, configLabel). An empty templatePath is the signal
|
||||
// to the provisioner that the existing volume is authoritative — the flow
|
||||
// that preserves user edits.
|
||||
//
|
||||
// Pure function: no writes, no DB access, no network. Safe to unit-test
|
||||
// with just a temp directory.
|
||||
func resolveRestartTemplate(configsDir, wsName, dbRuntime string, body restartTemplateInput) (templatePath, configLabel string) {
|
||||
template := body.Template
|
||||
|
||||
// Tier 2: name-based auto-match, gated on ApplyTemplate.
|
||||
if template == "" && body.ApplyTemplate {
|
||||
template = findTemplateByName(configsDir, wsName)
|
||||
}
|
||||
|
||||
// Tier 1 + 2 resolve via the same code path — validate + stat.
|
||||
if template != "" {
|
||||
candidatePath, resolveErr := resolveInsideRoot(configsDir, template)
|
||||
if resolveErr != nil {
|
||||
log.Printf("Restart: invalid template %q: %v — proceeding without it", template, resolveErr)
|
||||
template = ""
|
||||
} else if _, err := os.Stat(candidatePath); err == nil {
|
||||
return candidatePath, template
|
||||
} else {
|
||||
log.Printf("Restart: template %q dir not found — proceeding without it", template)
|
||||
}
|
||||
}
|
||||
|
||||
// Tier 3: #239 rebuild_config — org-templates as last-resort recovery.
|
||||
if body.RebuildConfig {
|
||||
if p, label := resolveOrgTemplate(configsDir, wsName); p != "" {
|
||||
log.Printf("Restart: rebuild_config — using org-template %s (%s)", label, wsName)
|
||||
return p, label
|
||||
}
|
||||
}
|
||||
|
||||
// Tier 4: runtime-default — apply_template=true + known runtime.
|
||||
// Use case: Canvas Config tab changed the runtime; we need the new
|
||||
// runtime's base files (entry point, Dockerfile, skill scaffolding)
|
||||
// because the existing volume was written by the old runtime.
|
||||
if body.ApplyTemplate && dbRuntime != "" {
|
||||
runtimeTemplate := filepath.Join(configsDir, dbRuntime+"-default")
|
||||
if _, err := os.Stat(runtimeTemplate); err == nil {
|
||||
label := dbRuntime + "-default"
|
||||
log.Printf("Restart: applying template %s (runtime change)", label)
|
||||
return runtimeTemplate, label
|
||||
}
|
||||
}
|
||||
|
||||
// Tier 5: reuse existing volume. This is the default, and the path
|
||||
// the Canvas Save+Restart flow MUST hit to preserve user edits.
|
||||
return "", "existing-volume"
|
||||
}
|
||||
178
workspace-server/internal/handlers/restart_template_test.go
Normal file
178
workspace-server/internal/handlers/restart_template_test.go
Normal file
@ -0,0 +1,178 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests for resolveRestartTemplate — the pure helper that implements the
|
||||
// priority chain documented on the function. Each test builds a minimal
|
||||
// temp configsDir, fabricates the specific precondition it exercises,
|
||||
// and asserts (templatePath, configLabel).
|
||||
//
|
||||
// The regression this suite locks in: a default restart (no flags) must
|
||||
// never auto-apply a template that happens to match the workspace name.
|
||||
// That was the "model reverts on Save+Restart" bug from
|
||||
// fix/restart-preserves-user-config.
|
||||
|
||||
// newTemplateDir makes a templates root with named subdirs, each holding
|
||||
// a minimal config.yaml so findTemplateByName's dir-scan path has
|
||||
// something to read. Returns the absolute root.
|
||||
func newTemplateDir(t *testing.T, names ...string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, n := range names {
|
||||
dir := filepath.Join(root, n)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", dir, err)
|
||||
}
|
||||
cfg := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(cfg, []byte("name: "+n+"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", cfg, err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_DefaultRestart_PreservesVolume is the
|
||||
// regression test for the Canvas Save+Restart bug. A workspace named
|
||||
// "Hermes Agent" normalises to "hermes-agent" — no dir match — but the
|
||||
// findTemplateByName second pass would also scan config.yaml's `name:`
|
||||
// field. We seed a template whose config.yaml DOES have the matching
|
||||
// name, exactly the worst case. Without apply_template, the helper
|
||||
// MUST still return empty templatePath.
|
||||
func TestResolveRestartTemplate_DefaultRestart_PreservesVolume(t *testing.T) {
|
||||
root := newTemplateDir(t, "hermes")
|
||||
// Overwrite config.yaml so the name-scan would hit:
|
||||
cfg := filepath.Join(root, "hermes", "config.yaml")
|
||||
if err := os.WriteFile(cfg, []byte("name: Hermes Agent\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Hermes Agent", "hermes", restartTemplateInput{
|
||||
// ApplyTemplate intentionally omitted — this is the default restart.
|
||||
})
|
||||
if path != "" {
|
||||
t.Errorf("default restart must NOT resolve a template; got path=%q", path)
|
||||
}
|
||||
if label != "existing-volume" {
|
||||
t.Errorf("expected 'existing-volume' label on default restart; got %q", label)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_ExplicitTemplate_AlwaysHonoured verifies
|
||||
// that passing Template by name works regardless of ApplyTemplate —
|
||||
// the caller named a template, that's unambiguous consent.
|
||||
func TestResolveRestartTemplate_ExplicitTemplate_AlwaysHonoured(t *testing.T) {
|
||||
root := newTemplateDir(t, "langgraph")
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Some Agent", "", restartTemplateInput{
|
||||
Template: "langgraph",
|
||||
})
|
||||
if path == "" || label != "langgraph" {
|
||||
t.Errorf("explicit template must resolve; got path=%q label=%q", path, label)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_ApplyTemplate_NameMatch verifies that
|
||||
// setting ApplyTemplate re-enables the name-based auto-match for
|
||||
// operators who actually want "reset this workspace to its template".
|
||||
func TestResolveRestartTemplate_ApplyTemplate_NameMatch(t *testing.T) {
|
||||
root := newTemplateDir(t, "hermes")
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Hermes", "", restartTemplateInput{
|
||||
ApplyTemplate: true,
|
||||
})
|
||||
if path == "" || label != "hermes" {
|
||||
t.Errorf("apply_template should name-match; got path=%q label=%q", path, label)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_ApplyTemplate_RuntimeDefault verifies the
|
||||
// runtime-change flow: when the Canvas Config tab changes the runtime,
|
||||
// the restart handler needs to lay down the new runtime's base files
|
||||
// via `<runtime>-default/`. Matches the existing behaviour comment.
|
||||
func TestResolveRestartTemplate_ApplyTemplate_RuntimeDefault(t *testing.T) {
|
||||
root := newTemplateDir(t, "langgraph-default")
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Some Workspace", "langgraph", restartTemplateInput{
|
||||
ApplyTemplate: true,
|
||||
})
|
||||
if path == "" || label != "langgraph-default" {
|
||||
t.Errorf("apply_template + dbRuntime should resolve runtime-default; got path=%q label=%q", path, label)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_ApplyTemplate_NoMatch_NoRuntime falls all
|
||||
// the way through to the reuse-volume path when neither name nor
|
||||
// runtime-default resolves.
|
||||
func TestResolveRestartTemplate_ApplyTemplate_NoMatch_NoRuntime(t *testing.T) {
|
||||
root := newTemplateDir(t) // empty templates dir
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Orphan", "", restartTemplateInput{
|
||||
ApplyTemplate: true,
|
||||
})
|
||||
if path != "" {
|
||||
t.Errorf("nothing to apply → expected empty path; got %q", path)
|
||||
}
|
||||
if label != "existing-volume" {
|
||||
t.Errorf("expected 'existing-volume' fallback; got %q", label)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_InvalidExplicitTemplate_ProceedsWithout
|
||||
// covers the defensive path where an explicit Template doesn't resolve
|
||||
// to a valid dir (e.g. traversal attempt, deleted template). The helper
|
||||
// must log + fall through, not crash or escape the root.
|
||||
func TestResolveRestartTemplate_InvalidExplicitTemplate_ProceedsWithout(t *testing.T) {
|
||||
root := newTemplateDir(t, "langgraph")
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Some Agent", "", restartTemplateInput{
|
||||
Template: "../../etc/passwd",
|
||||
})
|
||||
if path != "" {
|
||||
t.Errorf("traversal attempt must not resolve; got %q", path)
|
||||
}
|
||||
if label != "existing-volume" {
|
||||
t.Errorf("expected 'existing-volume' fallback on invalid template; got %q", label)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_NonExistentExplicitTemplate mirrors the
|
||||
// above but for a syntactically-valid name that simply doesn't exist
|
||||
// on disk (e.g. template was manually deleted). Must fall through.
|
||||
func TestResolveRestartTemplate_NonExistentExplicitTemplate(t *testing.T) {
|
||||
root := newTemplateDir(t, "langgraph")
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Some Agent", "", restartTemplateInput{
|
||||
Template: "deleted-template",
|
||||
})
|
||||
if path != "" {
|
||||
t.Errorf("missing template must not resolve; got %q", path)
|
||||
}
|
||||
if label != "existing-volume" {
|
||||
t.Errorf("expected 'existing-volume' fallback on missing template; got %q", label)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRestartTemplate_Priority_ExplicitBeatsApplyTemplate proves
|
||||
// that an explicit Template takes precedence over a name-based match.
|
||||
// Scenario: workspace "Hermes" with ApplyTemplate=true + explicit
|
||||
// Template="langgraph" — caller wants langgraph, not hermes.
|
||||
func TestResolveRestartTemplate_Priority_ExplicitBeatsApplyTemplate(t *testing.T) {
|
||||
root := newTemplateDir(t, "hermes", "langgraph")
|
||||
|
||||
path, label := resolveRestartTemplate(root, "Hermes", "", restartTemplateInput{
|
||||
Template: "langgraph",
|
||||
ApplyTemplate: true,
|
||||
})
|
||||
if label != "langgraph" {
|
||||
t.Errorf("explicit Template must win; got label=%q", label)
|
||||
}
|
||||
// Verify the path is actually inside the langgraph template dir
|
||||
expected := filepath.Join(root, "langgraph")
|
||||
if path != expected {
|
||||
t.Errorf("expected path %q, got %q", expected, path)
|
||||
}
|
||||
}
|
||||
@ -5,8 +5,6 @@ import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@ -127,53 +125,11 @@ func (h *WorkspaceHandler) Restart(c *gin.Context) {
|
||||
}
|
||||
c.ShouldBindJSON(&body)
|
||||
|
||||
// Resolve template path in priority order:
|
||||
// 1. Explicit template from request body
|
||||
// 2. Runtime-specific default template (e.g. claude-code-default/)
|
||||
// 3. Name-based match in templates directory
|
||||
// 4. No template — the volume already has configs from previous run
|
||||
var templatePath string
|
||||
var configFiles map[string][]byte
|
||||
configLabel := "existing-volume"
|
||||
|
||||
template := body.Template
|
||||
if template == "" {
|
||||
template = findTemplateByName(h.configsDir, wsName)
|
||||
}
|
||||
if template != "" {
|
||||
candidatePath, resolveErr := resolveInsideRoot(h.configsDir, template)
|
||||
if resolveErr != nil {
|
||||
log.Printf("Restart: invalid template %q: %v — proceeding without it", template, resolveErr)
|
||||
template = "" // clear so findTemplateByName fallback fires
|
||||
} else if _, err := os.Stat(candidatePath); err == nil {
|
||||
templatePath = candidatePath
|
||||
configLabel = template
|
||||
} else {
|
||||
log.Printf("Restart: template %q dir not found — proceeding without it", template)
|
||||
}
|
||||
}
|
||||
|
||||
// #239: rebuild_config=true — try org-templates as last-resort source so a
|
||||
// workspace with a destroyed config volume can self-recover without admin
|
||||
// intervention. Only fires when no other template was resolved above.
|
||||
if templatePath == "" && body.RebuildConfig {
|
||||
if p, label := resolveOrgTemplate(h.configsDir, wsName); p != "" {
|
||||
templatePath = p
|
||||
configLabel = label
|
||||
log.Printf("Restart: rebuild_config — using org-template %s for %s (%s)", label, wsName, id)
|
||||
}
|
||||
}
|
||||
|
||||
// #239: rebuild_config=true — try org-templates as last-resort source so a
|
||||
// workspace with a destroyed config volume can self-recover without admin
|
||||
// intervention. Only fires when no other template was resolved above.
|
||||
if templatePath == "" && body.RebuildConfig {
|
||||
if p, label := resolveOrgTemplate(h.configsDir, wsName); p != "" {
|
||||
templatePath = p
|
||||
configLabel = label
|
||||
log.Printf("Restart: rebuild_config — using org-template %s for %s (%s)", label, wsName, id)
|
||||
}
|
||||
}
|
||||
templatePath, configLabel := resolveRestartTemplate(h.configsDir, wsName, dbRuntime, restartTemplateInput{
|
||||
Template: body.Template,
|
||||
ApplyTemplate: body.ApplyTemplate,
|
||||
RebuildConfig: body.RebuildConfig,
|
||||
})
|
||||
|
||||
if templatePath == "" {
|
||||
log.Printf("Restart: reusing existing config volume for %s (%s)", wsName, id)
|
||||
@ -181,21 +137,10 @@ func (h *WorkspaceHandler) Restart(c *gin.Context) {
|
||||
log.Printf("Restart: using template %s for %s (%s)", templatePath, wsName, id)
|
||||
}
|
||||
|
||||
var configFiles map[string][]byte
|
||||
payload := models.CreateWorkspacePayload{Name: wsName, Tier: tier, Runtime: containerRuntime}
|
||||
log.Printf("Restart: workspace %s (%s) runtime=%q", wsName, id, containerRuntime)
|
||||
|
||||
// Apply runtime-default template ONLY when explicitly requested via "apply_template": true.
|
||||
// Use case: runtime was changed via Config tab — need new runtime's base files.
|
||||
// Normal restarts preserve existing config volume (user's model, skills, prompts).
|
||||
if templatePath == "" && body.ApplyTemplate && dbRuntime != "" {
|
||||
runtimeTemplate := filepath.Join(h.configsDir, dbRuntime+"-default")
|
||||
if _, err := os.Stat(runtimeTemplate); err == nil {
|
||||
templatePath = runtimeTemplate
|
||||
configLabel = dbRuntime + "-default"
|
||||
log.Printf("Restart: applying template %s (runtime change)", configLabel)
|
||||
}
|
||||
}
|
||||
|
||||
// #12: ?reset=true (or body.Reset) discards the claude-sessions volume
|
||||
// before restart, giving the agent a clean /root/.claude/sessions dir.
|
||||
resetClaudeSession := c.Query("reset") == "true" || body.Reset
|
||||
|
||||
160
workspace-server/internal/provisioner/backend_contract_test.go
Normal file
160
workspace-server/internal/provisioner/backend_contract_test.go
Normal file
@ -0,0 +1,160 @@
|
||||
package provisioner
|
||||
|
||||
// backend_contract_test.go — shared behavioral contract for the two
|
||||
// workspace backends (Docker + CPProvisioner).
|
||||
//
|
||||
// The two implementations today evolved independently — method names
|
||||
// line up on paper (Start/Stop/IsRunning/GetConsoleOutput) but the
|
||||
// semantics around error shapes, not-found cases, and cleanup can
|
||||
// drift because nothing holds them to a single interface. This file
|
||||
// establishes that contract.
|
||||
//
|
||||
// Structure:
|
||||
//
|
||||
// 1. `Backend` interface below — the union of methods both backends
|
||||
// must satisfy. Used as the compile-time gate that catches drift
|
||||
// (adding a method to one implementation without the other stops
|
||||
// compiling).
|
||||
//
|
||||
// 2. `runBackendContract(t, impl)` runs the same scenarios against
|
||||
// any `Backend` value. Each scenario is a table row; adding a
|
||||
// new behavior requires extending this one place, not two.
|
||||
//
|
||||
// 3. `TestDockerBackend_Contract` and `TestCPProvisionerBackend_
|
||||
// Contract` feed the real implementations through the shared
|
||||
// runner. They use lightweight fakes (nil Docker client, stub
|
||||
// HTTP server) so the tests run in CI without a real daemon or
|
||||
// control plane.
|
||||
//
|
||||
// This file is intentionally a skeleton — the scenarios list is short
|
||||
// today because we're establishing the pattern. Each follow-up PR
|
||||
// that touches a backend method should add its scenario here, not
|
||||
// bolt a new one-off test onto the implementation's own *_test.go.
|
||||
//
|
||||
// NON-GOAL: this is not a replacement for the existing per-backend
|
||||
// tests. Those cover implementation-specific concerns (Docker image
|
||||
// pull behavior, CP HTTP retry, etc.). This runner covers the
|
||||
// cross-backend behavior users care about.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Backend is the behavioral contract every workspace-provisioning
|
||||
// backend (Docker, CPProvisioner, future backends) must satisfy. Method
|
||||
// signatures here must match the actual implementations exactly — if
|
||||
// an implementation's signature drifts, Go compile-time catches it at
|
||||
// the assertion var blocks below.
|
||||
//
|
||||
// Kept minimal on purpose; expand only when a new cross-backend
|
||||
// behavior needs a contract test. Implementation-private methods stay
|
||||
// off this interface.
|
||||
type Backend interface {
|
||||
Start(ctx context.Context, cfg WorkspaceConfig) (string, error)
|
||||
Stop(ctx context.Context, workspaceID string) error
|
||||
IsRunning(ctx context.Context, workspaceID string) (bool, error)
|
||||
}
|
||||
|
||||
// Compile-time assertions — a method signature drift on either backend
|
||||
// makes this file fail to build, which is the whole point.
|
||||
var (
|
||||
_ Backend = (*Provisioner)(nil)
|
||||
_ Backend = (*CPProvisioner)(nil)
|
||||
)
|
||||
|
||||
// backendContractScenario is one behavior every backend must exhibit.
|
||||
type backendContractScenario struct {
|
||||
name string
|
||||
run func(t *testing.T, b Backend)
|
||||
}
|
||||
|
||||
// backendContractScenarios — extend this list when you add a new
|
||||
// cross-backend behavior. Each scenario runs against every registered
|
||||
// backend.
|
||||
//
|
||||
// Scenarios kept as methods on a closure so they can reference helpers
|
||||
// without polluting the package namespace.
|
||||
func backendContractScenarios() []backendContractScenario {
|
||||
return []backendContractScenario{
|
||||
{
|
||||
name: "IsRunning_UnknownWorkspace_ReturnsFalseAndNoError",
|
||||
// Contract: asking about a workspace the backend has never
|
||||
// seen must return (false, nil) — not a real error, not a
|
||||
// panic. Both current backends honor this today; this test
|
||||
// pins it so a future "optimization" doesn't break A2A's
|
||||
// alive-on-unknown path.
|
||||
run: func(t *testing.T, b Backend) {
|
||||
// Use a clearly-synthetic workspace ID that neither
|
||||
// backend should have state for.
|
||||
running, err := b.IsRunning(context.Background(), "contract-test-nonexistent-workspace-id")
|
||||
// The Docker backend returns (true, err) when it can't
|
||||
// reach the daemon — that's the "transient" contract
|
||||
// A2A relies on. The CP backend does the same when the
|
||||
// HTTP call fails. Both accept a transient-error shape.
|
||||
// For a not-found workspace both should return cleanly.
|
||||
// We allow either (false, nil) or (*, err) — the
|
||||
// contract prohibits (true, nil) for an unknown ID and
|
||||
// prohibits panic.
|
||||
_ = err
|
||||
_ = running
|
||||
// Contract assertion shape: we assert no panic (test
|
||||
// survives) + a recognizable return. Tightening this
|
||||
// requires deciding what the exact contract is; today
|
||||
// both backends do "best effort" lookup.
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Stop_UnknownWorkspace_IsIdempotent",
|
||||
// Contract: stopping a workspace that doesn't exist must
|
||||
// not error out. Important because the scheduler and the
|
||||
// orphan sweeper call Stop speculatively; if it errored on
|
||||
// unknown-id, every sweep would spam the logs and the
|
||||
// orphan path would never terminate cleanly.
|
||||
run: func(t *testing.T, b Backend) {
|
||||
err := b.Stop(context.Background(), "contract-test-nonexistent-workspace-id")
|
||||
if err != nil {
|
||||
t.Logf("Backend.Stop returned %v for unknown ID — acceptable as long as it doesn't panic, but ideally a no-op", err)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runBackendContract is the shared runner. Call this from each
|
||||
// implementation's contract test with a ready-to-use backend value.
|
||||
func runBackendContract(t *testing.T, backend Backend) {
|
||||
t.Helper()
|
||||
for _, sc := range backendContractScenarios() {
|
||||
t.Run(sc.name, func(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("Backend scenario %q panicked: %v", sc.name, r)
|
||||
}
|
||||
}()
|
||||
sc.run(t, backend)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDockerBackend_Contract feeds the Docker backend through the
|
||||
// shared runner. Skipped pending hardening: the scaffold exposed a
|
||||
// real bug — neither backend's Stop/IsRunning handles a nil underlying
|
||||
// client gracefully (both panic). Filing that as a separate issue;
|
||||
// once both backends return (*, error) instead of panicking, flip this
|
||||
// to t.Run and the contract scenarios exercise the fix.
|
||||
func TestDockerBackend_Contract(t *testing.T) {
|
||||
t.Skip("scaffolding only — unblock by hardening Provisioner.{Stop,IsRunning} against nil Docker client; see docs/architecture/backends.md drift risk #6")
|
||||
var p Provisioner
|
||||
runBackendContract(t, &p)
|
||||
}
|
||||
|
||||
// TestCPProvisionerBackend_Contract — same story as the Docker variant.
|
||||
// CPProvisioner panics on nil httpClient today; once that's hardened,
|
||||
// remove the Skip and this runner exercises both backends through a
|
||||
// single contract.
|
||||
func TestCPProvisionerBackend_Contract(t *testing.T) {
|
||||
t.Skip("scaffolding only — unblock by hardening CPProvisioner.{Stop,IsRunning} against nil httpClient; see docs/architecture/backends.md drift risk #6")
|
||||
var p CPProvisioner
|
||||
runBackendContract(t, &p)
|
||||
}
|
||||
@ -220,6 +220,9 @@ func Setup(hub *ws.Hub, broadcaster *events.Broadcaster, prov *provisioner.Provi
|
||||
|
||||
// Registry
|
||||
rh := handlers.NewRegistryHandler(broadcaster)
|
||||
// #1870 Phase 1: wire the queue drain hook so Heartbeat can dispatch
|
||||
// a queued A2A request when the workspace reports spare capacity.
|
||||
rh.SetQueueDrainFunc(wh.DrainQueueForWorkspace)
|
||||
r.POST("/registry/register", rh.Register)
|
||||
r.POST("/registry/heartbeat", rh.Heartbeat)
|
||||
r.POST("/registry/update-card", rh.UpdateCard)
|
||||
|
||||
1
workspace-server/migrations/042_a2a_queue.down.sql
Normal file
1
workspace-server/migrations/042_a2a_queue.down.sql
Normal file
@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS a2a_queue;
|
||||
53
workspace-server/migrations/042_a2a_queue.up.sql
Normal file
53
workspace-server/migrations/042_a2a_queue.up.sql
Normal file
@ -0,0 +1,53 @@
|
||||
-- #1870 Phase 1: TASK-level queue for A2A delegations that hit a busy target.
|
||||
--
|
||||
-- Before: when the target workspace's HTTP handler errors (agent busy
|
||||
-- mid-synthesis — single-threaded LLM loop), a2a_proxy_helpers.go returns
|
||||
-- 503 with a Retry-After hint, the caller logs activity_type='delegation'
|
||||
-- status='failed' and moves on. Delegations silently dropped; fan-out
|
||||
-- storms from leads reach ~70% drop rate.
|
||||
--
|
||||
-- After: same failure triggers an INSERT into a2a_queue with priority=TASK.
|
||||
-- Workspace's next heartbeat (up to 30s later) drains the queue if capacity
|
||||
-- allows. Proxy returns 202 Accepted with {"queued": true, "queue_id", ...}
|
||||
-- instead of 503, caller logs as dispatched-queued.
|
||||
--
|
||||
-- Phase 2 will add INFO (TTL) and CRITICAL (preempt) levels. This table's
|
||||
-- priority column is wide enough for all three from day one — no migration
|
||||
-- churn on next phase.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS a2a_queue (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
|
||||
caller_id uuid,
|
||||
priority smallint NOT NULL DEFAULT 50, -- 100=CRITICAL, 50=TASK, 10=INFO
|
||||
body jsonb NOT NULL,
|
||||
method text,
|
||||
idempotency_key text,
|
||||
enqueued_at timestamptz NOT NULL DEFAULT now(),
|
||||
dispatched_at timestamptz,
|
||||
completed_at timestamptz,
|
||||
expires_at timestamptz, -- TTL, for future INFO level
|
||||
attempts integer NOT NULL DEFAULT 0,
|
||||
status text NOT NULL DEFAULT 'queued' -- queued | dispatched | completed | dropped | failed
|
||||
CHECK (status IN ('queued','dispatched','completed','dropped','failed')),
|
||||
last_error text
|
||||
);
|
||||
|
||||
-- Primary drain-query index: pick oldest highest-priority queued item for a
|
||||
-- workspace. Partial index on status='queued' keeps the hot path tiny.
|
||||
CREATE INDEX IF NOT EXISTS idx_a2a_queue_dispatch
|
||||
ON a2a_queue (workspace_id, priority DESC, enqueued_at ASC)
|
||||
WHERE status = 'queued';
|
||||
|
||||
-- TTL index for future INFO cleanup (no-op today — expires_at is always NULL
|
||||
-- for TASK). Still worth creating now so Phase 2 doesn't need a migration.
|
||||
CREATE INDEX IF NOT EXISTS idx_a2a_queue_expiry
|
||||
ON a2a_queue (expires_at)
|
||||
WHERE status = 'queued' AND expires_at IS NOT NULL;
|
||||
|
||||
-- Idempotency: a caller retrying with the same idempotency_key should not
|
||||
-- double-enqueue. Partial unique index only on active queue entries so
|
||||
-- completed/dropped entries don't block future legitimate re-uses.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_a2a_queue_idempotency
|
||||
ON a2a_queue (workspace_id, idempotency_key)
|
||||
WHERE idempotency_key IS NOT NULL AND status IN ('queued','dispatched');
|
||||
Loading…
Reference in New Issue
Block a user