test: harden release validation live shards

This commit is contained in:
Peter Steinberger
2026-04-27 20:45:15 +01:00
parent f90972d942
commit 9645fe72c6
5 changed files with 85 additions and 59 deletions

View File

@@ -296,7 +296,7 @@ describeLive("openai plugin live", () => {
const collapsedText = text.replace(/[\s-]+/g, "");
expect(text.length).toBeGreaterThan(0);
expect(collapsedText).toContain("openclaw");
expect(text).toMatch(/\bok\b/);
expect(text).toMatch(/\bok(?:ay)?\b/);
}, 45_000);
it("opens OpenAI realtime STT before sending audio", async () => {

View File

@@ -86,6 +86,7 @@ function isGatewayBackendLiveTest(file) {
function isExtensionMediaLiveTest(file) {
return (
file === "extensions/music-generation-providers.live.test.ts" ||
file === "extensions/minimax/minimax.live.test.ts" ||
file === "extensions/openai/openai-tts.live.test.ts" ||
file === "extensions/video-generation-providers.live.test.ts" ||
file === "extensions/volcengine/tts.live.test.ts" ||

View File

@@ -303,6 +303,15 @@ function isUnsupportedPlanErrorMessage(raw: string): boolean {
return /current token plan (?:does )?not support (?:this )?model/i.test(raw);
}
function isOpenRouterOpaqueBadRequestErrorMessage(raw: string): boolean {
const msg = raw.toLowerCase();
return (
msg.includes("provider returned error") &&
msg.includes('"code":400') &&
msg.includes('"msg":"bad request"')
);
}
describe("isUnsupportedPlanErrorMessage", () => {
it("matches provider plan-gated models", () => {
expect(isUnsupportedPlanErrorMessage("current token plan does not support this model")).toBe(
@@ -313,6 +322,17 @@ describe("isUnsupportedPlanErrorMessage", () => {
});
});
describe("isOpenRouterOpaqueBadRequestErrorMessage", () => {
it("matches opaque OpenRouter upstream bad requests", () => {
expect(
isOpenRouterOpaqueBadRequestErrorMessage(
'Error: 400 Provider returned error {"code":400,"msg":"bad request","request_id":"abc"}',
),
).toBe(true);
expect(isOpenRouterOpaqueBadRequestErrorMessage("Error: 400 bad request")).toBe(false);
});
});
function toInt(value: string | undefined, fallback: number): number {
const trimmed = value?.trim();
if (!trimmed) {
@@ -1151,6 +1171,15 @@ describeLive("live models (profile keys)", () => {
logProgress(`${progressLabel}: skip (provider unavailable)`);
break;
}
if (
allowNotFoundSkip &&
model.provider === "openrouter" &&
isOpenRouterOpaqueBadRequestErrorMessage(message)
) {
skipped.push({ model: id, reason: message });
logProgress(`${progressLabel}: skip (openrouter upstream bad request)`);
break;
}
if (allowNotFoundSkip && isModelNotFoundErrorMessage(message)) {
skipped.push({ model: id, reason: message });
logProgress(`${progressLabel}: skip (model not found)`);

View File

@@ -1,75 +1,70 @@
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { clearPluginDiscoveryCache } from "../plugins/discovery.js";
import { clearPluginManifestRegistryCache } from "../plugins/manifest-registry.js";
import { beforeEach, describe, expect, it, vi } from "vitest";
const runChannelPluginStartupMaintenanceMock = vi.fn().mockResolvedValue(undefined);
const runChannelPluginStartupMaintenanceMock = vi.hoisted(() =>
vi.fn().mockResolvedValue(undefined),
);
vi.mock("../channels/plugins/lifecycle-startup.js", () => ({
runChannelPluginStartupMaintenance: runChannelPluginStartupMaintenanceMock,
runChannelPluginStartupMaintenance: (params: unknown) =>
runChannelPluginStartupMaintenanceMock(params),
}));
import {
getFreePort,
installGatewayTestHooks,
startGatewayServer,
testState,
} from "./test-helpers.js";
vi.mock("../agents/agent-scope.js", () => ({
resolveAgentWorkspaceDir: () => "/workspace",
resolveDefaultAgentId: () => "default",
}));
installGatewayTestHooks({ scope: "suite" });
vi.mock("../agents/subagent-registry.js", () => ({
initSubagentRegistry: vi.fn(),
}));
describe("gateway startup channel maintenance wiring", () => {
it("runs startup channel maintenance with the resolved startup config", async () => {
const previousBundledPluginsDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR;
const previousSkipChannels = process.env.OPENCLAW_SKIP_CHANNELS;
process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = path.resolve(process.cwd(), "extensions");
process.env.OPENCLAW_SKIP_CHANNELS = "0";
clearPluginDiscoveryCache();
clearPluginManifestRegistryCache();
beforeEach(() => {
vi.resetModules();
runChannelPluginStartupMaintenanceMock.mockClear();
});
testState.channelsConfig = {
matrix: {
homeserver: "https://matrix.example.org",
userId: "@bot:example.org",
accessToken: "tok-123",
it("runs startup channel maintenance with the resolved startup config", async () => {
const { prepareGatewayPluginBootstrap } = await import("./server-startup-plugins.js");
await prepareGatewayPluginBootstrap({
cfgAtStart: {
plugins: { enabled: true },
},
};
startupRuntimeConfig: {
plugins: { enabled: true },
channels: {
matrix: {
homeserver: "https://matrix.example.org",
userId: "@bot:example.org",
accessToken: "tok-123",
},
},
},
minimalTestGateway: true,
log: {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
},
});
let server: Awaited<ReturnType<typeof startGatewayServer>> | undefined;
try {
server = await startGatewayServer(await getFreePort());
expect(runChannelPluginStartupMaintenanceMock).toHaveBeenCalledTimes(1);
expect(runChannelPluginStartupMaintenanceMock).toHaveBeenCalledWith(
expect.objectContaining({
cfg: expect.objectContaining({
channels: expect.objectContaining({
matrix: expect.objectContaining({
homeserver: "https://matrix.example.org",
userId: "@bot:example.org",
accessToken: "tok-123",
}),
expect(runChannelPluginStartupMaintenanceMock).toHaveBeenCalledTimes(1);
expect(runChannelPluginStartupMaintenanceMock).toHaveBeenCalledWith(
expect.objectContaining({
cfg: expect.objectContaining({
channels: expect.objectContaining({
matrix: expect.objectContaining({
homeserver: "https://matrix.example.org",
userId: "@bot:example.org",
accessToken: "tok-123",
}),
}),
env: process.env,
log: expect.anything(),
}),
);
} finally {
await server?.close();
if (previousBundledPluginsDir === undefined) {
delete process.env.OPENCLAW_BUNDLED_PLUGINS_DIR;
} else {
process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = previousBundledPluginsDir;
}
if (previousSkipChannels === undefined) {
delete process.env.OPENCLAW_SKIP_CHANNELS;
} else {
process.env.OPENCLAW_SKIP_CHANNELS = previousSkipChannels;
}
clearPluginDiscoveryCache();
clearPluginManifestRegistryCache();
}
env: process.env,
log: expect.anything(),
}),
);
});
});

View File

@@ -41,6 +41,7 @@ describe("scripts/test-live-shard", () => {
expect(selectLiveShardFiles("native-live-extensions-media", allFiles)).toEqual(
expect.arrayContaining([
"extensions/openai/openai-tts.live.test.ts",
"extensions/minimax/minimax.live.test.ts",
"extensions/music-generation-providers.live.test.ts",
"extensions/video-generation-providers.live.test.ts",
"extensions/volcengine/tts.live.test.ts",