From 1ae051ec95968dd141e046e5a15b232a1ca37377 Mon Sep 17 00:00:00 2001 From: Hongming Wang Date: Sun, 26 Apr 2026 10:03:46 -0700 Subject: [PATCH] test(e2e): add 'Invalid API key' regression assertion to staging A2A check (#1900) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The staging E2E suite already grep's for 5 known regression patterns in the A2A response (hermes-agent 401, model_not_found, Encrypted content, Unknown provider, hermes-agent unreachable). The comment block at lines 386-395 lists "Invalid API key" as the signal for the CP #238 boot-event 401 race + stale OPENAI_API_KEY paths, but the explicit grep was never added — meaning a regression in that class would slip through the generic `error|exception` catch-all. Closes the gap with one specific-pattern check that fails loud with the relevant bug references in the message. Verified `bash -n` clean; pre-existing shellcheck SC2015 at line 88 is unrelated. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/e2e/test_staging_full_saas.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/e2e/test_staging_full_saas.sh b/tests/e2e/test_staging_full_saas.sh index e498ed46..ce3be39b 100755 --- a/tests/e2e/test_staging_full_saas.sh +++ b/tests/e2e/test_staging_full_saas.sh @@ -410,6 +410,13 @@ fi if echo "$AGENT_TEXT" | grep -qF "Unknown provider"; then fail "A2A — REGRESSION: install.sh set PROVIDER to a value not in hermes's registry. Run 'hermes doctor' on the workspace to see valid values. Raw: $AGENT_TEXT" fi +# "Invalid API key" — the comment block lists this as a CP #238 race +# (tenant auth chain) signal but the grep was missing. Caller-side +# 401's containing this exact phrase don't match the generic +# "error|exception" catch-all below, so they'd slip through. +if echo "$AGENT_TEXT" | grep -qF "Invalid API key"; then + fail "A2A — REGRESSION: tenant auth chain returned 'Invalid API key'. Likely CP boot-event 401 race (CP #238) or stale OPENAI_API_KEY in the runtime env. Raw: $AGENT_TEXT" +fi # Generic catch-all — falls through if none of the known regressions hit. if echo "$AGENT_TEXT" | grep -qiE "error|exception"; then fail "A2A returned an error-shaped response: $AGENT_TEXT"