From a2af8054e1a480a4171c944cb41847e3285927e4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 15:11:41 +0100 Subject: [PATCH] test: harden live release checks --- src/agents/zai.live.test.ts | 21 +++++++++++++------ ...gateway-codex-harness.live-helpers.test.ts | 10 +++++++++ .../gateway-codex-harness.live-helpers.ts | 7 +++++-- .../gateway-models.profiles.live.test.ts | 8 +++++++ test/gateway.multi.e2e.test.ts | 2 +- test/helpers/gateway-e2e-harness.ts | 9 +++++++- 6 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/agents/zai.live.test.ts b/src/agents/zai.live.test.ts index 2ce4765b684..1861216fd0d 100644 --- a/src/agents/zai.live.test.ts +++ b/src/agents/zai.live.test.ts @@ -8,6 +8,7 @@ import { const ZAI_KEY = process.env.ZAI_API_KEY ?? process.env.Z_AI_API_KEY ?? ""; const LIVE = isLiveTestEnabled(["ZAI_LIVE_TEST"]); +const ZAI_LIVE_TIMEOUT_MS = 45_000; const describeLive = LIVE && ZAI_KEY ? describe : describe.skip; @@ -25,11 +26,19 @@ async function expectModelReturnsAssistantText(modelId: "glm-5" | "glm-5.1") { } describeLive("zai live", () => { - it("returns assistant text", async () => { - await expectModelReturnsAssistantText("glm-5"); - }, 20000); + it( + "returns assistant text", + async () => { + await expectModelReturnsAssistantText("glm-5"); + }, + ZAI_LIVE_TIMEOUT_MS, + ); - it("glm-5.1 returns assistant text", async () => { - await expectModelReturnsAssistantText("glm-5.1"); - }, 20000); + it( + "glm-5.1 returns assistant text", + async () => { + await expectModelReturnsAssistantText("glm-5.1"); + }, + ZAI_LIVE_TIMEOUT_MS, + ); }); diff --git a/src/gateway/gateway-codex-harness.live-helpers.test.ts b/src/gateway/gateway-codex-harness.live-helpers.test.ts index e93a776f9a6..f5c71fcfbd0 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.test.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.test.ts @@ -27,6 +27,16 @@ describe("gateway codex harness live helpers", () => { expect(isExpectedCodexStatusCommandText(text)).toBe(true); }); + it("accepts current app-server status prose without the OpenClaw prefix", () => { + const text = [ + "Status: running on `openai/gpt-5.5` in `/tmp/openclaw-live-codex-harness/workspace/dev`.", + "", + "Context is at 22k / 272k tokens, with no compactions. There’s 1 active task: `/codex status`.", + ].join("\n"); + + expect(isExpectedCodexStatusCommandText(text)).toBe(true); + }); + it("rejects status prose for a different codex session", () => { const text = "OpenClaw is running on `openai/gpt-5.5` with low reasoning/text settings. Context is at `22k/272k` tokens, no compactions, and the current session is `agent:dev:other`."; diff --git a/src/gateway/gateway-codex-harness.live-helpers.ts b/src/gateway/gateway-codex-harness.live-helpers.ts index 19cbfb52808..1f2fb915522 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.ts @@ -88,14 +88,17 @@ export const EXPECTED_CODEX_STATUS_COMMAND_TEXT = [ export function isExpectedCodexStatusCommandText(text: string): boolean { const normalized = text.toLowerCase(); const mentionsOpenClawStatus = - normalized.includes("openclaw is running on") || normalized.includes("openclaw status:"); + normalized.includes("openclaw is running on") || + normalized.includes("openclaw status:") || + normalized.includes("status: running on"); const mentionsHarnessSession = normalized.includes("session: `agent:dev:live-codex-harness`") || normalized.includes("session: agent:dev:live-codex-harness") || normalized.includes("session `agent:dev:live-codex-harness`") || normalized.includes("current session is `agent:dev:live-codex-harness`") || normalized.includes("current session is agent:dev:live-codex-harness") || - (normalized.includes("session context") && normalized.includes("active task: `/codex status`")); + ((normalized.includes("session context") || normalized.includes("context is at")) && + normalized.includes("active task: `/codex status`")); const mentionsModel = normalized.includes("`openai/") || normalized.includes(" openai/") || diff --git a/src/gateway/gateway-models.profiles.live.test.ts b/src/gateway/gateway-models.profiles.live.test.ts index 3df09c7712a..cecc380a6d7 100644 --- a/src/gateway/gateway-models.profiles.live.test.ts +++ b/src/gateway/gateway-models.profiles.live.test.ts @@ -84,6 +84,8 @@ const GATEWAY_LIVE_STRIP_SCAFFOLDING_MODEL_KEYS = new Set([ ]); const GATEWAY_LIVE_EXEC_READ_NONCE_MISS_SKIP_MODEL_KEYS = new Set([ "fireworks/accounts/fireworks/models/glm-5", + "fireworks/accounts/fireworks/models/kimi-k2p5", + "fireworks/accounts/fireworks/models/kimi-k2p6", "fireworks/accounts/fireworks/routers/kimi-k2p5-turbo", "google/gemini-3.1-flash-lite-preview", ]); @@ -502,6 +504,12 @@ describe("shouldSkipExecReadNonceMissForLiveModel", () => { expect( shouldSkipExecReadNonceMissForLiveModel("fireworks/accounts/fireworks/models/glm-5"), ).toBe(true); + expect( + shouldSkipExecReadNonceMissForLiveModel("fireworks/accounts/fireworks/models/kimi-k2p5"), + ).toBe(true); + expect( + shouldSkipExecReadNonceMissForLiveModel("fireworks/accounts/fireworks/models/kimi-k2p6"), + ).toBe(true); expect( shouldSkipExecReadNonceMissForLiveModel( "fireworks/accounts/fireworks/routers/kimi-k2p5-turbo", diff --git a/test/gateway.multi.e2e.test.ts b/test/gateway.multi.e2e.test.ts index 01e8eea84ec..85bbe977a5f 100644 --- a/test/gateway.multi.e2e.test.ts +++ b/test/gateway.multi.e2e.test.ts @@ -116,7 +116,7 @@ describe("gateway multi-instance e2e", () => { events: chatEvents, runId: String(runId), sessionKey, - timeoutMs: 45_000, + timeoutMs: 90_000, }); const finalText = extractFirstTextBlock(finalEvent.message); expect(typeof finalText).toBe("string"); diff --git a/test/helpers/gateway-e2e-harness.ts b/test/helpers/gateway-e2e-harness.ts index d2a870136ab..4f3e922b5fc 100644 --- a/test/helpers/gateway-e2e-harness.ts +++ b/test/helpers/gateway-e2e-harness.ts @@ -465,5 +465,12 @@ export async function waitForChatFinalEvent(params: { } await sleep(20); } - throw new Error(`timeout waiting for final chat event (runId=${params.runId})`); + const observed = params.events + .filter((evt) => evt.runId === params.runId || evt.sessionKey === params.sessionKey) + .map((evt) => `${evt.runId ?? "no-run"}:${evt.sessionKey ?? "no-session"}:${evt.state}`) + .slice(-10) + .join(", "); + throw new Error( + `timeout waiting for final chat event (runId=${params.runId}, sessionKey=${params.sessionKey}, observed=${observed || "none"})`, + ); }