test: harden live release checks

This commit is contained in:
Peter Steinberger
2026-04-27 15:11:41 +01:00
parent 016a0b4de9
commit a2af8054e1
6 changed files with 47 additions and 10 deletions

View File

@@ -8,6 +8,7 @@ import {
const ZAI_KEY = process.env.ZAI_API_KEY ?? process.env.Z_AI_API_KEY ?? "";
const LIVE = isLiveTestEnabled(["ZAI_LIVE_TEST"]);
const ZAI_LIVE_TIMEOUT_MS = 45_000;
const describeLive = LIVE && ZAI_KEY ? describe : describe.skip;
@@ -25,11 +26,19 @@ async function expectModelReturnsAssistantText(modelId: "glm-5" | "glm-5.1") {
}
describeLive("zai live", () => {
it("returns assistant text", async () => {
await expectModelReturnsAssistantText("glm-5");
}, 20000);
it(
"returns assistant text",
async () => {
await expectModelReturnsAssistantText("glm-5");
},
ZAI_LIVE_TIMEOUT_MS,
);
it("glm-5.1 returns assistant text", async () => {
await expectModelReturnsAssistantText("glm-5.1");
}, 20000);
it(
"glm-5.1 returns assistant text",
async () => {
await expectModelReturnsAssistantText("glm-5.1");
},
ZAI_LIVE_TIMEOUT_MS,
);
});

View File

@@ -27,6 +27,16 @@ describe("gateway codex harness live helpers", () => {
expect(isExpectedCodexStatusCommandText(text)).toBe(true);
});
it("accepts current app-server status prose without the OpenClaw prefix", () => {
const text = [
"Status: running on `openai/gpt-5.5` in `/tmp/openclaw-live-codex-harness/workspace/dev`.",
"",
"Context is at 22k / 272k tokens, with no compactions. Theres 1 active task: `/codex status`.",
].join("\n");
expect(isExpectedCodexStatusCommandText(text)).toBe(true);
});
it("rejects status prose for a different codex session", () => {
const text =
"OpenClaw is running on `openai/gpt-5.5` with low reasoning/text settings. Context is at `22k/272k` tokens, no compactions, and the current session is `agent:dev:other`.";

View File

@@ -88,14 +88,17 @@ export const EXPECTED_CODEX_STATUS_COMMAND_TEXT = [
export function isExpectedCodexStatusCommandText(text: string): boolean {
const normalized = text.toLowerCase();
const mentionsOpenClawStatus =
normalized.includes("openclaw is running on") || normalized.includes("openclaw status:");
normalized.includes("openclaw is running on") ||
normalized.includes("openclaw status:") ||
normalized.includes("status: running on");
const mentionsHarnessSession =
normalized.includes("session: `agent:dev:live-codex-harness`") ||
normalized.includes("session: agent:dev:live-codex-harness") ||
normalized.includes("session `agent:dev:live-codex-harness`") ||
normalized.includes("current session is `agent:dev:live-codex-harness`") ||
normalized.includes("current session is agent:dev:live-codex-harness") ||
(normalized.includes("session context") && normalized.includes("active task: `/codex status`"));
((normalized.includes("session context") || normalized.includes("context is at")) &&
normalized.includes("active task: `/codex status`"));
const mentionsModel =
normalized.includes("`openai/") ||
normalized.includes(" openai/") ||

View File

@@ -84,6 +84,8 @@ const GATEWAY_LIVE_STRIP_SCAFFOLDING_MODEL_KEYS = new Set([
]);
const GATEWAY_LIVE_EXEC_READ_NONCE_MISS_SKIP_MODEL_KEYS = new Set([
"fireworks/accounts/fireworks/models/glm-5",
"fireworks/accounts/fireworks/models/kimi-k2p5",
"fireworks/accounts/fireworks/models/kimi-k2p6",
"fireworks/accounts/fireworks/routers/kimi-k2p5-turbo",
"google/gemini-3.1-flash-lite-preview",
]);
@@ -502,6 +504,12 @@ describe("shouldSkipExecReadNonceMissForLiveModel", () => {
expect(
shouldSkipExecReadNonceMissForLiveModel("fireworks/accounts/fireworks/models/glm-5"),
).toBe(true);
expect(
shouldSkipExecReadNonceMissForLiveModel("fireworks/accounts/fireworks/models/kimi-k2p5"),
).toBe(true);
expect(
shouldSkipExecReadNonceMissForLiveModel("fireworks/accounts/fireworks/models/kimi-k2p6"),
).toBe(true);
expect(
shouldSkipExecReadNonceMissForLiveModel(
"fireworks/accounts/fireworks/routers/kimi-k2p5-turbo",

View File

@@ -116,7 +116,7 @@ describe("gateway multi-instance e2e", () => {
events: chatEvents,
runId: String(runId),
sessionKey,
timeoutMs: 45_000,
timeoutMs: 90_000,
});
const finalText = extractFirstTextBlock(finalEvent.message);
expect(typeof finalText).toBe("string");

View File

@@ -465,5 +465,12 @@ export async function waitForChatFinalEvent(params: {
}
await sleep(20);
}
throw new Error(`timeout waiting for final chat event (runId=${params.runId})`);
const observed = params.events
.filter((evt) => evt.runId === params.runId || evt.sessionKey === params.sessionKey)
.map((evt) => `${evt.runId ?? "no-run"}:${evt.sessionKey ?? "no-session"}:${evt.state}`)
.slice(-10)
.join(", ");
throw new Error(
`timeout waiting for final chat event (runId=${params.runId}, sessionKey=${params.sessionKey}, observed=${observed || "none"})`,
);
}