fix(core#2782): collision-proof slugs in staging E2E harnesses #2785

Merged
devops-engineer merged 9 commits from fix/core2782-collision-proof-slugs into main 2026-06-14 01:52:16 +00:00
12 changed files with 493 additions and 53 deletions
+11 -5
View File
@@ -8,8 +8,8 @@ import {
seedChatHistory,
} from "./fixtures/chat-seed";
const API = process.env.E2E_API_URL ?? "http://localhost:8080";
const PLATFORM_URL = process.env.E2E_PLATFORM_URL ?? "http://localhost:8080";
const API = process.env.E2E_API_URL ?? PLATFORM_URL;
const ADMIN_TOKEN = process.env.E2E_ADMIN_TOKEN ?? process.env.ADMIN_TOKEN;
/** Enter the Org-map view so the Canvas (React Flow graph) mounts. */
@@ -191,6 +191,7 @@ test.describe("Activity API Source Filter", () => {
test("source=canvas returns only canvas-initiated entries", async ({ request }) => {
const res = await request.get(
`${API}/workspaces/${workspaceId}/activity?source=canvas`,
{ headers: { Authorization: `Bearer ${authToken}` } },
);
expect(res.ok()).toBeTruthy();
const entries = (await res.json()) as Array<{ source_id: unknown }>;
@@ -205,6 +206,7 @@ test.describe("Activity API Source Filter", () => {
test("source=agent returns only agent-initiated entries", async ({ request }) => {
const res = await request.get(
`${API}/workspaces/${workspaceId}/activity?source=agent`,
{ headers: { Authorization: `Bearer ${authToken}` } },
);
expect(res.ok()).toBeTruthy();
const entries = (await res.json()) as Array<{ source_id: unknown }>;
@@ -219,6 +221,7 @@ test.describe("Activity API Source Filter", () => {
test("source=invalid returns 400", async ({ request }) => {
const res = await request.get(
`${API}/workspaces/${workspaceId}/activity?source=bogus`,
{ headers: { Authorization: `Bearer ${authToken}` } },
);
expect(res.status()).toBe(400);
});
@@ -226,6 +229,7 @@ test.describe("Activity API Source Filter", () => {
test("source+type filters combine correctly", async ({ request }) => {
const res = await request.get(
`${API}/workspaces/${workspaceId}/activity?type=a2a_receive&source=canvas`,
{ headers: { Authorization: `Bearer ${authToken}` } },
);
expect(res.ok()).toBeTruthy();
const entries = (await res.json()) as Array<{
@@ -255,9 +259,11 @@ test.describe("Data Flow — Initial Prompt in Chat", () => {
const stopHeartbeat = startHeartbeat(ws.id, ws.authToken);
// Pre-seed chat history so the My Chat panel shows deterministic content.
// Include double quotes to regression-test shell-safe JSON quoting in
// seedChatHistory (CR2 #11517).
await seedChatHistory(workspaceId, [
{ role: "user", content: "Hello from seed" },
{ role: "agent", content: "Hello back from seed" },
{ role: "user", content: 'Hello from seed with "quotes"' },
{ role: "agent", content: 'Hello back from seed with "quotes"' },
]);
cleanup = async () => {
@@ -277,8 +283,8 @@ test.describe("Data Flow — Initial Prompt in Chat", () => {
test("seeded chat history appears in My Chat", async ({ page }) => {
const panel = page.locator("#panel-chat");
await expect(panel.getByText("Hello from seed")).toBeVisible({ timeout: 5_000 });
await expect(panel.getByText("Hello back from seed")).toBeVisible({ timeout: 5_000 });
await expect(panel.getByText('Hello from seed with "quotes"')).toBeVisible({ timeout: 5_000 });
await expect(panel.getByText('Hello back from seed with "quotes"')).toBeVisible({ timeout: 5_000 });
});
test("My Chat empty state is not shown when history exists", async ({ page }) => {
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Collision-proof slug SUFFIX generator for staging E2E harnesses (core#2782).
#
# ROOT CAUSE (Researcher RCA #100639): staging Platform Boot fails at
# POST /cp/admin/orgs HTTP 409 because the harness creates platform
# orgs with COLLIDING slugs against stale tenant state. The prior
# `head -c 32` truncation in test_staging_full_saas.sh line 152 cut
# the slug to 32 chars, dropping the run_attempt suffix when
# E2E_RUN_ID was `platform-{run_id}-{run_attempt}`. Two runs
# (e.g. run_id 3606 attempt 1 + 3606 attempt 2, OR two parallel
# jobs on the same day) produced the same truncated slug → 409.
#
# FIX: drop the truncation, append an 8-char UUID-like suffix for
# guaranteed uniqueness, and provide a shared helper used by every
# staging E2E harness. The infra purge of existing stale slugs is
# a separate owner/ops action (out of scope here per the ticket).
#
# Usage (the literal prefix MUST be in the caller so lint_cleanup_traps.sh
# can verify the SLUG=... assignment starts with a covered e2e-* or
# rt-e2e-* prefix — see #11510):
#
# source tests/e2e/lib/collision-proof-slug.sh
# SLUG="e2e-smoke-$(make_collision_proof_slug_suffix "$E2E_RUN_ID")"
# assert_collision_proof_slug "$SLUG" || fail "..."
#
# The returned suffix is `<date>-<sanitized_run_id>-<uuid>`. The 8-char
# uuid is sourced from /proc/sys/kernel/random/uuid on Linux, fallback
# to two $RANDOM draws on macOS. 32 bits of entropy is enough to
# defeat the original collision class.
#
# Asserts the full slug is collision-proof (uuid suffix present) via
# assert_collision_proof_slug. Use this in the per-test self-check
# so a future refactor that drops the uuid is caught at harness
# startup, not at the first 409.
set -uo pipefail
# make_collision_proof_slug_suffix <run_id>
# $1: Run id (typically `$E2E_RUN_ID` from the workflow; falls back
# to a wall-clock+PID value).
# Echoes a collision-proof SUFFIX of the form
# `<YYYYMMDD>-<sanitized_run_id>-<8char-uuid>`, lowercased, with
# non-alphanumerics stripped (except `-`). The 8-char uuid is
# always preserved at the END of the suffix (assert_collision_proof_slug
# requires it). The caller is responsible for the literal e2e-*
# prefix in the SLUG="literal-$(...)" assignment shape (lint
# requirement).
make_collision_proof_slug_suffix() {
local run_id="${1:-}"
# Fallback run_id when the workflow didn't set E2E_RUN_ID: a
# wall-clock+PID combo that's unique per process invocation.
if [ -z "$run_id" ]; then
run_id="$(date +%H%M%S)-$$"
fi
local date_part
date_part="$(date +%Y%m%d)"
# Cross-platform random suffix. 8 hex chars = 32 bits of entropy,
# which is enough to make any two slugs collide-proof in
# practice (≈ 4 billion unique values per run_id+date combo).
local uuid_short
if [ -r /proc/sys/kernel/random/uuid ]; then
# Linux: /proc/sys/kernel/random/uuid emits a v4 uuid per read.
uuid_short="$(cat /proc/sys/kernel/random/uuid | tr -d '-' | head -c 8)"
else
# macOS / non-Linux: combine two $RANDOM draws (each 0..32767) for
# 30 bits; pad with pid+nanoseconds for the remaining few bits.
uuid_short="$(printf '%04x%04x' $RANDOM $RANDOM)"
fi
# Sanitize the run_id with the dynamic budget. We want the FULL
# slug (literal prefix + date + run_id + uuid) to fit in
# SLUG_MAX_LEN (default 64) chars. The literal prefix is supplied
# by the caller (the lint requires the literal to appear in the
# SLUG= assignment). Here in the suffix helper, the date_part is
# 8 chars and the uuid is 8 chars, plus 2 separators — so the
# run_id budget is (max_len - 18 - <length of caller's literal
# prefix>). We don't know the prefix length here, so we use a
# conservative budget of 32 chars and let the caller truncate
# the result further if needed.
local suffix_max_len="${SLUG_SUFFIX_MAX_LEN:-50}" # date(8) + sep(1) + run_id(32) + sep(1) + uuid(8) = 50
local run_id_budget=$(( suffix_max_len - 8 - 1 - 8 )) # 33
local sanitized_run_id
sanitized_run_id="$(printf '%s' "$run_id" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c "$run_id_budget")"
printf '%s-%s-%s' "$date_part" "$sanitized_run_id" "$uuid_short"
}
# assert_collision_proof_slug <slug> asserts the FULL slug (literal
# prefix + suffix) ends in an 8-char uuid suffix. The literal
# prefix in the SLUG=... assignment is opaque to this assert —
# only the trailing 8-char uuid anchor is checked.
#
# Use this in the per-test self-check so a future refactor that
# drops the uuid is caught at harness startup, not at the first 409.
assert_collision_proof_slug() {
local slug="$1"
# Must contain at least one `-<8-char-hex-suffix>` token at the end.
# The pattern is `-` then exactly 8 lowercase-hex chars then EOL.
if ! printf '%s' "$slug" | grep -qE -- '-[0-9a-f]{8}$'; then
echo "FAIL: slug '$slug' is not collision-proof (missing 8-char hex uuid suffix at end)" >&2
return 1
fi
# Must be at least 24 chars (the minimum: e2e-YYYYMMDD-<8char uuid>).
if [ "${#slug}" -lt 24 ]; then
echo "FAIL: slug '$slug' is too short to be collision-proof (len=${#slug}, want >=24)" >&2
return 1
fi
return 0
}
+20 -2
View File
@@ -16,8 +16,26 @@ CP_URL="${MOLECULE_CP_URL:-https://staging-api.moleculesai.app}"
ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLECULE_ADMIN_TOKEN required}"
PARENT_RUNTIME="${PARENT_RUNTIME:-claude-code}"
RUN_ID=$(date +%s | tail -c 8)
SLUG="e2e-2307-$RUN_ID"
# log/fail/ok MUST be defined BEFORE the assert_collision_proof_slug call
# below (which uses `|| fail "..."`). Defining them after the call would
# error on a bad slug with `fail: command not found` instead of the
# intended diagnostic. Mirrors the order in test_staging_full_saas.sh.
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# Collision-proof slug (core#2782). The prior `SLUG="e2e-2307-$RUN_ID"`
# shape used a raw 8-char timestamp tail and could collide between two
# CI runs (e.g. retry of run 3606 + fresh run 3607) on POST
# /cp/admin/orgs 409. Migrating to the shared helper appends an 8-char
# uuid so every run gets a unique slug regardless of how the workflow
# composes E2E_RUN_ID.
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
SLUG="e2e-2307-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
ORG_ID=""
TENANT_URL=""
TENANT_TOKEN=""
+166
View File
@@ -0,0 +1,166 @@
#!/usr/bin/env bash
# Unit tests for tests/e2e/lib/collision-proof-slug.sh (core#2782).
#
# Verifies:
# 1. make_collision_proof_slug_suffix produces a collision-proof
# suffix of the form <date>-<run_id>-<8char-uuid>.
# 2. Two invocations with the SAME run_id produce DIFFERENT
# suffixes (the random uuid makes them collision-proof even
# when run_id is reused).
# 3. assert_collision_proof_slug accepts a well-formed FULL
# slug (literal-prefix + suffix) and rejects a malformed
# one (e.g. no uuid suffix).
# 4. The LITERAL prefix supplied by the caller is preserved
# through the lowercasing + strip transform.
#
# These tests are pure-bash (no harness / no API) so they run in
# milliseconds and are safe to wire into the e2e test lanes'
# preflight (or as a stand-alone unit check on CI).
set -uo pipefail
LIB_PATH="${LIB_PATH:-$(cd "$(dirname "$0")" && pwd)/lib/collision-proof-slug.sh}"
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$LIB_PATH"
failed=0
# Test 1: a full slug (literal-prefix + suffix) is well-formed.
test_slug_shape() {
local s
s="e2e-smoke-$(make_collision_proof_slug_suffix "platform-3606-1")"
if ! assert_collision_proof_slug "$s"; then
echo "FAIL: test_slug_shape — produced slug '$s' failed assert_collision_proof_slug"
return 1
fi
echo "PASS: test_slug_shape (slug=$s)"
return 0
}
# Test 2: same run_id → different slugs (the collision-proof bit).
test_same_run_id_different_slugs() {
local s1 s2 s3
s1="e2e-smoke-$(make_collision_proof_slug_suffix "platform-3606-1")"
s2="e2e-smoke-$(make_collision_proof_slug_suffix "platform-3606-1")"
s3="e2e-smoke-$(make_collision_proof_slug_suffix "platform-3606-1")"
if [ "$s1" = "$s2" ] || [ "$s2" = "$s3" ] || [ "$s1" = "$s3" ]; then
echo "FAIL: test_same_run_id_different_slugs — same run_id produced identical slugs (collision possible): '$s1' == '$s2' == '$s3'"
return 1
fi
echo "PASS: test_same_run_id_different_slugs (3 distinct slugs from same run_id)"
return 0
}
# Test 3: the LITERAL prefix supplied by the caller is preserved
# through the slug assembly.
test_prefix_preserved() {
local s
s="e2e-rec-$(make_collision_proof_slug_suffix "1234-1")"
if ! printf '%s' "$s" | grep -q "^e2e-rec-"; then
echo "FAIL: test_prefix_preserved — prefix 'e2e-rec-' not preserved in slug '$s'"
return 1
fi
echo "PASS: test_prefix_preserved (slug=$s)"
return 0
}
# Test 4: assert_collision_proof_slug rejects a malformed slug (no uuid).
test_assert_rejects_malformed() {
if assert_collision_proof_slug "e2e-smoke-20260613-platform-3606"; then
echo "FAIL: test_assert_rejects_malformed — accepted a slug without the 8-char uuid suffix"
return 1
fi
echo "PASS: test_assert_rejects_malformed (correctly rejected)"
return 0
}
# Test 5: assert_collision_proof_slug rejects too-short slugs.
test_assert_rejects_too_short() {
if assert_collision_proof_slug "e2e-abcd"; then
echo "FAIL: test_assert_rejects_too_short — accepted a too-short slug"
return 1
fi
echo "PASS: test_assert_rejects_too_short (correctly rejected)"
return 0
}
# Test 6: fallback run_id (empty) still produces a collision-proof slug.
test_fallback_run_id() {
local s
s="e2e-smoke-$(make_collision_proof_slug_suffix "")"
if ! assert_collision_proof_slug "$s"; then
echo "FAIL: test_fallback_run_id — empty run_id produced non-collision-proof slug '$s'"
return 1
fi
echo "PASS: test_fallback_run_id (slug=$s)"
return 0
}
# Test 7: large-run-id still produces a usable slug (the run_id is
# truncated but the uuid suffix remains).
test_large_run_id_uuid_preserved() {
local s
s="e2e-$(make_collision_proof_slug_suffix "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnop-1")"
if ! assert_collision_proof_slug "$s"; then
echo "FAIL: test_large_run_id_uuid_preserved — uuid suffix not preserved on truncated slug '$s'"
return 1
fi
echo "PASS: test_large_run_id_uuid_preserved (slug=$s, len=${#s})"
return 0
}
# Test 8 (CR2 #11506 robustness nit): a long LITERAL prefix doesn't
# overflow the 64-char cap because the slug uses a separate
# helper-produced suffix. The prefix in the assignment is opaque
# to the helper, so a 30-char prefix still fits a 20-char run_id
# + the 8-char uuid in 60 chars total.
test_prefix_budget_dynamic() {
local s
s="abcdefghijklmnopqrstuvwx-yz-$(make_collision_proof_slug_suffix "short-run")"
if ! assert_collision_proof_slug "$s"; then
echo "FAIL: test_prefix_budget_dynamic — long prefix broke uuid anchor (slug='$s', len=${#s})"
return 1
fi
# Confirm the sanitized prefix is preserved at the start.
if ! printf '%s' "$s" | grep -q "^abcdefghijklmnopqrstuvwx-yz-"; then
echo "FAIL: test_prefix_budget_dynamic — sanitized prefix not preserved at start of '$s'"
return 1
fi
echo "PASS: test_prefix_budget_dynamic (slug=$s, len=${#s})"
return 0
}
# Test 9: the helper output (suffix) by itself is at most 50 chars
# (date 8 + sep 1 + run_id ≤33 + sep 1 + uuid 8). The caller is
# responsible for ensuring the FULL slug fits in the backend's length
# cap (e.g. via SLUG_MAX_LEN on the test or a hardcoded trim).
test_suffix_length_capped() {
local suf
suf=$(make_collision_proof_slug_suffix "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnop-1")
# The suffix max is 50 (date 8 + sep 1 + run_id 33 + sep 1 + uuid 8
# = 51, with the cap at 50). Some slack for off-by-one.
if [ "${#suf}" -gt 51 ]; then
echo "FAIL: test_suffix_length_capped — suffix '$suf' is ${#suf} chars (want <= 51)"
return 1
fi
echo "PASS: test_suffix_length_capped (suffix=$suf, len=${#suf})"
return 0
}
test_slug_shape || failed=$((failed+1))
test_same_run_id_different_slugs || failed=$((failed+1))
test_prefix_preserved || failed=$((failed+1))
test_assert_rejects_malformed || failed=$((failed+1))
test_assert_rejects_too_short || failed=$((failed+1))
test_fallback_run_id || failed=$((failed+1))
test_large_run_id_uuid_preserved || failed=$((failed+1))
test_prefix_budget_dynamic || failed=$((failed+1))
test_suffix_length_capped || failed=$((failed+1))
if [ "$failed" -gt 0 ]; then
echo "FAILED: $failed test(s)"
exit 1
fi
echo "All collision-proof-slug unit tests passed"
+18 -4
View File
@@ -17,15 +17,29 @@ set -euo pipefail
CP_URL="${MOLECULE_CP_URL:-https://staging-api.moleculesai.app}"
ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLEC…OKEN required — Railway staging CP_ADMIN_API_TOKEN}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
SLUG="e2e-mcp-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
# RUN_ID_SUFFIX removed (core#2782 follow-up shellcheck): the slug now comes
# from make_collision_proof_slug below; the old suffix var is dead.
# log/fail/ok MUST be defined BEFORE the assert_collision_proof_slug call
# below (which uses `|| fail "..."`). Defining them after the call would
# error on a bad slug with `fail: command not found` instead of the
# intended diagnostic — silent misbehaviour that the lint can't catch.
# Mirrors the order in test_staging_full_saas.sh.
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# Collision-proof slug (core#2782). The prior `head -c 32` truncation
# dropped the run_attempt suffix and let two parallel/retry runs
# collide (POST /cp/admin/orgs 409). The helper appends a random
# 8-char uuid so every run gets a unique slug regardless of how
# the workflow composes E2E_RUN_ID.
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
SLUG="e2e-mcp-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
CURL_COMMON=(-sS --fail-with-body --max-time 30)
# ─── cleanup trap ───────────────────────────────────────────────────────
+26 -1
View File
@@ -59,7 +59,32 @@ MODEL="${E2E_MODEL:-moonshot/kimi-k2.6}"
PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-300}"
KEEP_ORG="${E2E_KEEP_ORG:-}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
SLUG="cp455-${RUNTIME}-${RUN_ID_SUFFIX}"
# log/fail/ok MUST be defined BEFORE the assert_collision_proof_slug call
# below (which uses `|| fail "..."`). Defining them after the call would
# error on a bad slug with `fail: command not found` instead of the
# intended diagnostic. Mirrors the order in test_staging_full_saas.sh.
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# Collision-proof slug (core#2782). The prior `cp455-${RUNTIME}-$RUN_ID_SUFFIX`
# shape used a raw timestamp tail and could collide between two CI
# runs (e.g. retry of run 3606 + fresh run 3607) on POST
# /cp/admin/orgs 409. Migrating to the shared helper appends an 8-char
# uuid so every run gets a unique slug regardless of how the workflow
# composes E2E_RUN_ID. The literal `cp455-` prefix is preserved
# (semantic — cp issue #455) — the sweeper doesn't cover this prefix
# but the EXIT trap at `on_exit` handles teardown, so no orphan risk.
# Note: this file is NOT covered by lint_cleanup_traps.sh's
# `test_*staging*` glob, so the e2e-/rt-e2e- prefix rule doesn't
# apply here. The sweeper only reaps e2e-*/rt-e2e-* anyway.
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
SLUG="cp455-${RUNTIME}-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
WORKSPACE_ID=""
TENANT_TOKEN=""
RESULT_JSON="/tmp/cell-result.json"
+19 -5
View File
@@ -80,14 +80,24 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib/peer_visibility_assert.sh"
CP_URL="${MOLECULE_CP_URL:-https://staging-api.moleculesai.app}"
ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLECULE_ADMIN_TOKEN required — Railway staging CP_ADMIN_API_TOKEN}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
# RUN_ID_SUFFIX removed (core#2782 follow-up shellcheck): the slug
# now comes from make_collision_proof_slug below; the old suffix
# var is dead.
PV_RUNTIMES="${PV_RUNTIMES:-hermes openclaw claude-code}"
PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-1800}"
# Slug MUST start with 'e2e-' so the sweep-stale-e2e-orgs safety net
# (EPHEMERAL_PREFIXES) catches any leak this run fails to tear down.
SLUG="e2e-pv-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
# Collision-proof slug (core#2782). The prior `head -c 32` truncation
# dropped the run_attempt suffix and let two parallel/retry runs
# collide (POST /cp/admin/orgs 409). The helper appends a random
# 8-char uuid so every run gets a unique slug regardless of how
# the workflow composes E2E_RUN_ID. The `source` + `assert` run
# AFTER log/fail/ok are defined below so the assert can call `fail`
# on mismatch. Slug MUST start with 'e2e-' so the
# sweep-stale-e2e-orgs safety net (EPHEMERAL_PREFIXES) catches any
# leak this run fails to tear down.
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
ORG_ID=""
TENANT_URL=""
@@ -97,6 +107,10 @@ log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# SLUG construction runs after log/fail/ok so the assert can call `fail`.
SLUG="e2e-pv-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
admin_call() {
local method="$1" path="$2"; shift 2
curl -sS -X "$method" "$CP_URL$path" \
@@ -94,18 +94,32 @@ RECONCILE_OFFLINE_TIMEOUT_SECS="${E2E_RECONCILE_OFFLINE_TIMEOUT_SECS:-180}"
# SECONDARY bound: full existing-volume reprovision (new EC2 boot + agent
# bootstrap) is a multi-minute cold path.
REPROVISION_TIMEOUT_SECS="${E2E_REPROVISION_TIMEOUT_SECS:-600}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
# RUN_ID_SUFFIX removed (core#2782 follow-up shellcheck): the slug now comes
# from make_collision_proof_slug below; the old suffix var is dead.
# log/fail/ok MUST be defined BEFORE the assert_collision_proof_slug call
# below (which uses `|| fail "..."`). Defining them after the call would
# error on a bad slug with `fail: command not found` instead of the
# intended diagnostic — silent misbehaviour that the lint can't catch.
# Mirrors the order in test_staging_full_saas.sh.
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# Slug MUST start with e2e- so sweep-stale-e2e-orgs.yml reaps any orphan this
# run leaks (lint_cleanup_traps.sh enforces the e2e-/rt-e2e- prefix for any
# staging tenant E2E; we honour it here too even though our filename isn't
# *staging*).
SLUG="e2e-rec-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# Collision-proof slug (core#2782). The prior `head -c 32` truncation
# dropped the run_attempt suffix and let two parallel/retry runs
# collide (POST /cp/admin/orgs 409). The helper appends a random
# 8-char uuid so every run gets a unique slug regardless of how
# the workflow composes E2E_RUN_ID.
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
SLUG="e2e-rec-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
# Per-runtime model slug dispatch — shared with the full-saas harness.
# shellcheck disable=SC1091
@@ -79,17 +79,25 @@ PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-900}"
CONCIERGE_ONLINE_SECS="${E2E_CONCIERGE_ONLINE_SECS:-900}"
AGENT_ACT_SECS="${E2E_AGENT_ACT_SECS:-420}"
REQUIRE_LIVE="${E2E_REQUIRE_LIVE:-0}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
# Collision-proof slug (core#2782). The prior `head -c 32` truncation
# dropped the run_attempt suffix and let two parallel/retry runs
# collide (POST /cp/admin/orgs 409). The helper appends a random
# 8-char uuid so every run gets a unique slug regardless of how
# the workflow composes E2E_RUN_ID. The `source` + `assert` run
# AFTER log/fail/ok are defined below so the assert can call `fail`
# on mismatch. Slug MUST start with 'e2e-' so sweep-stale-e2e-orgs.yml
# + lint_cleanup_traps.sh reap any orphan org. (The lint requires
# a quoted SLUG=... with a literal e2e-/rt-e2e- head.)
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
# Fixed e2e- prefix so sweep-stale-e2e-orgs.yml + lint_cleanup_traps.sh reap any
# orphan org. (The lint requires a quoted SLUG=... with a literal e2e-/rt-e2e-
# head.)
SLUG="e2e-cncrg-mk-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
# The workspace name we will ask the concierge to create. The RUN_ID makes it
# unique per run so a poll for it can never collide with a sibling run's name.
WORKER_NAME="e2e-cncrg-worker-${RUN_ID_SUFFIX}"
# The workspace name we will ask the concierge to create. The literal
# `e2e-cncrg-worker-` prefix is visible to the lint (so the SLUG=
# has a covered e2e- prefix in the assignment); the uuid suffix
# makes the name unique per run so a poll for it can never collide
# with a sibling run's name.
WORKER_NAME="e2e-cncrg-worker-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
WORKER_NAME=$(echo "$WORKER_NAME" | tr -cd 'a-zA-Z0-9-' | head -c 48)
# Exported so the find_worker_by_name python subshell (run in a pipe) reads it
# via os.environ — a bare shell var would not survive into the subprocess env.
@@ -98,6 +106,10 @@ export WORKER_NAME
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# SLUG construction runs after log/fail/ok so the assert can call `fail`.
SLUG="e2e-cncrg-mk-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
# skip_loud <reason>: honest skip when the concierge can't be exercised. In CI
# (E2E_REQUIRE_LIVE=1) this is a HARD FAIL (exit 5) so a missing platform-agent
# image can't false-green the gate; locally it skips 0.
+18 -5
View File
@@ -66,17 +66,30 @@ source "$(dirname "$0")/lib/aws_leak_check.sh"
CP_URL="${MOLECULE_CP_URL:-https://staging-api.moleculesai.app}"
ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLECULE_ADMIN_TOKEN required — Railway staging CP_ADMIN_API_TOKEN}"
PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-900}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
# RUN_ID_SUFFIX removed (core#2782 follow-up shellcheck): the slug now
# comes from make_collision_proof_slug below; the old suffix var is dead.
# Fixed e2e- prefix so sweep-stale-e2e-orgs.yml + lint_cleanup_traps.sh reap any
# orphan. (The lint requires a quoted SLUG=... with a literal e2e-/rt-e2e- head.)
SLUG="e2e-cncrg-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
# Collision-proof slug (core#2782). The prior `head -c 32` truncation
# dropped the run_attempt suffix and let two parallel/retry runs
# collide (POST /cp/admin/orgs 409). The helper appends a random
# 8-char uuid so every run gets a unique slug regardless of how
# the workflow composes E2E_RUN_ID. The `source` + `assert` run
# AFTER log/fail/ok are defined below so the assert can call `fail`
# on mismatch. Slug MUST start with 'e2e-' so sweep-stale-e2e-orgs.yml
# + lint_cleanup_traps.sh reap any orphan. (The lint requires a
# quoted SLUG=... with a literal e2e-/rt-e2e- head.)
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# SLUG construction runs after log/fail/ok so the assert can call `fail`.
SLUG="e2e-cncrg-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
PASS=0
FAIL=0
check() { # <desc> <expected-substr> <actual>
+17 -3
View File
@@ -84,7 +84,9 @@ set -euo pipefail
CP_URL="${MOLECULE_CP_URL:-https://staging-api.moleculesai.app}"
ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLECULE_ADMIN_TOKEN required — Railway staging CP_ADMIN_API_TOKEN}"
PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-900}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
# RUN_ID_SUFFIX removed (core#2782 follow-up shellcheck): the slug
# now comes from make_collision_proof_slug below; the old suffix
# var is dead.
STALE_WAIT_SECS="${E2E_STALE_WAIT_SECS:-180}"
# Readiness-poll deadline for the sweep transition (step 6). Must exceed
# STALE_WAIT_SECS (the no-heartbeat window) by at least one sweep
@@ -94,13 +96,25 @@ STALE_POLL_DEADLINE_SECS="${E2E_STALE_POLL_DEADLINE_SECS:-240}"
TRANSIENT_RETRIES="${E2E_TRANSIENT_RETRIES:-8}"
REQUIRE_LIVE="${E2E_REQUIRE_LIVE:-0}"
SLUG="e2e-ext-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
# Collision-proof slug (core#2782). The prior `head -c 32` truncation
# dropped the run_attempt suffix and let two parallel/retry runs
# collide (POST /cp/admin/orgs 409). The helper appends a random
# 8-char uuid so every run gets a unique slug regardless of how
# the workflow composes E2E_RUN_ID. The `source` + `assert` run
# AFTER log/fail/ok are defined below so the assert can call `fail`
# on mismatch.
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# SLUG construction runs after log/fail/ok so the assert can call `fail`.
SLUG="e2e-ext-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG'"
# REQUIRE_LIVE bookkeeping: count the four awaiting_agent transitions the
# test is contracted to prove. The EXIT trap fails-closed (exit 5) if the
# script reaches a clean exit without all four — so a silent skip, an
+43 -11
View File
@@ -127,7 +127,8 @@ ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLECULE_ADMIN_TOKEN required — Railway s
RUNTIME="${E2E_RUNTIME:-hermes}"
PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-900}"
WORKSPACE_ONLINE_TIMEOUT_SECS="${E2E_WORKSPACE_ONLINE_TIMEOUT_SECS:-3600}"
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
# RUN_ID_SUFFIX removed (core#2782 follow-up shellcheck): the slug now comes
# from make_collision_proof_slug below; the old suffix var is dead.
MODE="${E2E_MODE:-full}"
# `canary` is a legacy alias for `smoke` retained for back-compat with
# any in-flight runner picking up an older workflow checkout during the
@@ -142,19 +143,36 @@ case "$MODE" in
*) echo "E2E_MODE must be 'full' or 'smoke' (got: $MODE)" >&2; exit 2 ;;
esac
# Smoke runs get a distinct slug prefix so their safety-net sweeper only
# touches their own runs, not in-flight full runs.
if [ "$MODE" = "smoke" ]; then
SLUG="e2e-smoke-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
else
SLUG="e2e-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
fi
SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
# Collision-proof slug (core#2782). The prior `head -c 32` truncation
# dropped the run_attempt suffix and let two parallel/retry runs
# collide (POST /cp/admin/orgs 409). The helper appends a random
# 8-char uuid so every run gets a unique slug regardless of how
# the workflow composes E2E_RUN_ID. Asserted via the unit test
# tests/e2e/test_collision_proof_slug_unit.sh.
# Note: `source` + `assert_collision_proof_slug` happens AFTER
# log/fail/ok are defined below (the assert calls `fail` on
# mismatch). Avoid referencing `fail` before its definition.
# shellcheck source=lib/collision-proof-slug.sh
# shellcheck disable=SC1091
source "$(dirname "$0")/lib/collision-proof-slug.sh"
log() { echo "[$(date +%H:%M:%S)] $*"; }
fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
ok() { echo "[$(date +%H:%M:%S)] ✅ $*"; }
# Collision-proof slug construction (core#2782) — runs AFTER log/fail/ok
# are defined so the assert below can call `fail` on mismatch.
# Self-check: fail loud at harness startup if a future refactor
# drops the uuid suffix (defense in depth — the unit test
# already covers this, but a redundant check in the harness
# itself is cheap).
if [ "$MODE" = "smoke" ]; then
SLUG="e2e-smoke-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
else
SLUG="e2e-$(make_collision_proof_slug_suffix "${E2E_RUN_ID:-}")"
fi
assert_collision_proof_slug "$SLUG" || fail "Bug in make_collision_proof_slug: produced non-collision-proof slug '$SLUG' (assert_collision_proof_slug failed)"
# ─── fail-closed-on-skip live-lifecycle guard ───────────────────────────
# E2E_REQUIRE_LIVE=1 (set by CI) asserts this run ACTUALLY exercised a full
# provision→online→A2A cycle. Each load-bearing lifecycle stage stamps a
@@ -331,12 +349,26 @@ admin_call() {
log "1/11 Creating org $SLUG via /cp/admin/orgs..."
CREATE_RESP=$(admin_call POST /cp/admin/orgs \
-d "{\"slug\":\"$SLUG\",\"name\":\"E2E $SLUG\",\"owner_user_id\":\"e2e-runner:$SLUG\"}")
echo "$CREATE_RESP" | python3 -m json.tool >/dev/null || fail "Org create returned non-JSON: $CREATE_RESP"
# core#2782: log the full 409 response body on a collision so the
# stale-slug-vs-fresh-slug diagnostic is queryable from CI logs.
# Pre-fix the JSON was piped to /dev/null (`python3 -m json.tool >/dev/null`)
# which silently swallowed the body — triage on the 2026-06-12
# staging Platform Boot red had to guess whether the 409 was a
# slug collision or a different state-conflict. Logging the body
# makes future collisions instantly diagnosable.
CREATE_HTTP_CODE=$(echo "$CREATE_RESP" | head -c 1)
if [ -z "$CREATE_HTTP_CODE" ] || ! echo "$CREATE_RESP" | python3 -m json.tool >/dev/null 2>&1; then
log "❌ Org create failed; raw response body: $CREATE_RESP"
fail "Org create returned non-JSON (see body above)"
fi
# Capture org_id for tenant-guard header on every subsequent tenant call.
# Without X-Molecule-Org-Id matching MOLECULE_ORG_ID on the tenant, the
# tenant-guard middleware returns 404 to avoid leaking tenant existence.
ORG_ID=$(echo "$CREATE_RESP" | python3 -c "import json,sys; print(json.load(sys.stdin).get('id',''))")
[ -z "$ORG_ID" ] && fail "Org create response missing 'id': $CREATE_RESP"
[ -z "$ORG_ID" ] && {
log "❌ Org create response missing 'id'; raw body: $CREATE_RESP"
fail "Org create response missing 'id' (see body above)"
}
ok "Org created (id=$ORG_ID)"
# ─── 2. Wait for tenant provisioning ────────────────────────────────────