diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index b7b35de6681..9e977168900 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -665,7 +665,6 @@ jobs: matrix_args=( --repo-root . \ - --output-dir "${output_dir}" \ --provider-mode mock-openai \ --model mock-openai/gpt-5.5 \ --alt-model mock-openai/gpt-5.5-alt \ @@ -676,7 +675,17 @@ jobs: matrix_args+=(--fail-fast) fi - pnpm openclaw qa matrix "${matrix_args[@]}" + for attempt in 1 2; do + attempt_output_dir="${output_dir}/attempt-${attempt}" + if pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"; then + exit 0 + fi + if [[ "${attempt}" == "2" ]]; then + exit 1 + fi + echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2 + sleep 10 + done - name: Upload Matrix QA artifacts if: always() @@ -751,15 +760,25 @@ jobs: output_dir=".artifacts/qa-e2e/telegram-live-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - pnpm openclaw qa telegram \ - --repo-root . \ - --output-dir "${output_dir}" \ - --provider-mode mock-openai \ - --model mock-openai/gpt-5.5 \ - --alt-model mock-openai/gpt-5.5-alt \ - --fast \ - --credential-source convex \ - --credential-role ci + for attempt in 1 2; do + attempt_output_dir="${output_dir}/attempt-${attempt}" + if pnpm openclaw qa telegram \ + --repo-root . \ + --output-dir "${attempt_output_dir}" \ + --provider-mode mock-openai \ + --model mock-openai/gpt-5.5 \ + --alt-model mock-openai/gpt-5.5-alt \ + --fast \ + --credential-source convex \ + --credential-role ci; then + exit 0 + fi + if [[ "${attempt}" == "2" ]]; then + exit 1 + fi + echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2 + sleep 10 + done - name: Upload Telegram QA artifacts if: always() diff --git a/extensions/openai/openai.live.test.ts b/extensions/openai/openai.live.test.ts index f4ab84d3aac..6038e8b25be 100644 --- a/extensions/openai/openai.live.test.ts +++ b/extensions/openai/openai.live.test.ts @@ -13,6 +13,11 @@ import { } from "openclaw/plugin-sdk/plugin-test-runtime"; import { runRealtimeSttLiveTest } from "openclaw/plugin-sdk/provider-test-contracts"; import { getRuntimeConfig } from "openclaw/plugin-sdk/runtime-config-snapshot"; +import { + isOverloadedErrorMessage, + isServerErrorMessage, + isTimeoutErrorMessage, +} from "openclaw/plugin-sdk/test-env"; import { describe, expect, it } from "vitest"; import plugin from "./index.js"; @@ -99,6 +104,21 @@ function createReferencePng(): Buffer { return encodePngRgba(buf, width, height); } +function formatLiveOpenAIError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function resolveLiveOpenAISkipReason(error: unknown): string | null { + const message = formatLiveOpenAIError(error); + if (isTimeoutErrorMessage(message) || /timed out|operation was aborted/i.test(message)) { + return "provider timeout"; + } + if (isOverloadedErrorMessage(message) || isServerErrorMessage(message)) { + return "provider outage"; + } + return null; +} + function createLiveConfig(): OpenClawConfig { const cfg = getRuntimeConfig(); return { @@ -436,22 +456,36 @@ describeLive("openai plugin live", () => { const agentDir = await createTempAgentDir(); try { - const description = await mediaProvider.describeImage?.({ - buffer: createReferencePng(), - fileName: "reference.png", - mime: "image/png", - prompt: "Reply with one lowercase word for the dominant center color.", - timeoutMs: 120_000, - agentDir, - cfg, - authStore: EMPTY_AUTH_STORE, - model: LIVE_VISION_MODEL, - provider: "openai", - }); + let description: + | Awaited>> + | undefined; + try { + description = await mediaProvider.describeImage?.({ + buffer: createReferencePng(), + fileName: "reference.png", + mime: "image/png", + prompt: "Reply with one lowercase word for the dominant center color.", + timeoutMs: 45_000, + agentDir, + cfg, + authStore: EMPTY_AUTH_STORE, + model: LIVE_VISION_MODEL, + provider: "openai", + }); + } catch (err) { + const skipReason = resolveLiveOpenAISkipReason(err); + if (skipReason) { + console.warn( + `[live:openai] image description skipped: ${skipReason}: ${formatLiveOpenAIError(err)}`, + ); + return; + } + throw err; + } expect((description?.text ?? "").toLowerCase()).toContain("orange"); } finally { await fs.rm(agentDir, { recursive: true, force: true }); } - }, 180_000); + }, 75_000); }); diff --git a/scripts/lib/ci-node-test-plan.mjs b/scripts/lib/ci-node-test-plan.mjs index c9cdd2e10d3..4e3b78e3714 100644 --- a/scripts/lib/ci-node-test-plan.mjs +++ b/scripts/lib/ci-node-test-plan.mjs @@ -1,5 +1,6 @@ import { existsSync, readdirSync } from "node:fs"; import { join, relative } from "node:path"; +import { commandsLightTestFiles } from "../../test/vitest/vitest.commands-light-paths.mjs"; import { fullSuiteVitestShards } from "../../test/vitest/vitest.test-shards.mjs"; const EXCLUDED_FULL_SUITE_SHARDS = new Set([ @@ -84,6 +85,63 @@ function createAutoReplyReplySplitShards() { .filter((shard) => shard.includePatterns.length > 0); } +function resolveCommandShardName(file) { + const name = relative("src/commands", file).replaceAll("\\", "/"); + if (name.startsWith("agent") || name.startsWith("channel") || name === "message.test.ts") { + return "agentic-commands-agent-channel"; + } + if (name.startsWith("doctor")) { + if (name.startsWith("doctor/shared/") || name.startsWith("doctor/")) { + return "agentic-commands-doctor-shared"; + } + return "agentic-commands-doctor"; + } + if ( + name.startsWith("auth-choice") || + name.startsWith("configure") || + name.startsWith("onboard") || + name === "setup.test.ts" + ) { + return "agentic-commands-onboard-config"; + } + if ( + name.startsWith("models/") || + name === "model-picker.test.ts" || + name === "openai-model-default.test.ts" + ) { + return "agentic-commands-models"; + } + return "agentic-commands-status-tools"; +} + +function createAgenticCommandSplitShards() { + const commandsLightTests = new Set(commandsLightTestFiles); + const groups = new Map(); + for (const file of listTestFiles("src/commands")) { + if (commandsLightTests.has(file)) { + continue; + } + const shardName = resolveCommandShardName(file); + groups.set(shardName, [...(groups.get(shardName) ?? []), file]); + } + + return [ + "agentic-commands-agent-channel", + "agentic-commands-doctor", + "agentic-commands-doctor-shared", + "agentic-commands-models", + "agentic-commands-onboard-config", + "agentic-commands-status-tools", + ] + .map((shardName) => ({ + configs: ["test/vitest/vitest.commands.config.ts"], + includePatterns: groups.get(shardName) ?? [], + requiresDist: false, + shardName, + })) + .filter((shard) => shard.includePatterns.length > 0); +} + const SPLIT_NODE_SHARDS = new Map([ [ "core-unit-fast", @@ -179,15 +237,19 @@ const SPLIT_NODE_SHARDS = new Map([ runner: "blacksmith-4vcpu-ubuntu-2404", }, { - shardName: "agentic-commands", + shardName: "agentic-cli", + configs: ["test/vitest/vitest.cli.config.ts"], + requiresDist: false, + }, + { + shardName: "agentic-command-support", configs: [ - "test/vitest/vitest.cli.config.ts", "test/vitest/vitest.commands-light.config.ts", - "test/vitest/vitest.commands.config.ts", "test/vitest/vitest.daemon.config.ts", ], requiresDist: false, }, + ...createAgenticCommandSplitShards(), { shardName: "agentic-agents", configs: [ diff --git a/src/media-understanding/image.test.ts b/src/media-understanding/image.test.ts index f4ba23bf27f..ca37d6442f4 100644 --- a/src/media-understanding/image.test.ts +++ b/src/media-understanding/image.test.ts @@ -553,6 +553,28 @@ describe("describeImageWithModel", () => { expect(options?.signal?.aborted).toBe(true); }); + it("rejects when image runtime setup exceeds the request timeout", async () => { + vi.useFakeTimers(); + ensureOpenClawModelsJsonMock.mockImplementationOnce(() => new Promise(() => {})); + + const result = describeImageWithModel({ + cfg: {}, + agentDir: "/tmp/openclaw-agent", + provider: "openai", + model: "gpt-5.4-mini", + buffer: Buffer.from("png-bytes"), + fileName: "image.png", + mime: "image/png", + prompt: "Describe the image.", + timeoutMs: 25, + }); + + const assertion = expect(result).rejects.toThrow("image description timed out after 25ms"); + await vi.advanceTimersByTimeAsync(25); + await assertion; + expect(completeMock).not.toHaveBeenCalled(); + }); + it("normalizes deprecated google flash ids before lookup and keeps profile auth selection", async () => { const findMock = vi.fn((provider: string, modelId: string) => { expect(provider).toBe("google"); diff --git a/src/media-understanding/image.ts b/src/media-understanding/image.ts index 3e2a89536b0..0184c2a11cb 100644 --- a/src/media-understanding/image.ts +++ b/src/media-understanding/image.ts @@ -349,11 +349,17 @@ async function describeImagesWithModelInternal( options: { onPayload?: ProviderStreamOptions["onPayload"] } = {}, ): Promise { const prompt = params.prompt ?? "Describe the image."; + const startedAtMs = Date.now(); + const controller = new AbortController(); let apiKey: string; let model: Model | undefined; try { - const resolved = await resolveImageRuntime(params); + const resolved = await withImageDescriptionTimeout({ + controller, + timeoutMs: resolveImageDescriptionTimeoutMs(params.timeoutMs, startedAtMs), + task: resolveImageRuntime(params), + }); apiKey = resolved.apiKey; model = resolved.model; } catch (err) { @@ -391,8 +397,6 @@ async function describeImagesWithModelInternal( const context = buildImageContext(prompt, params.images, { promptInUserContent: shouldPlaceImagePromptInUserContent(model), }); - const startedAtMs = Date.now(); - const controller = new AbortController(); const maxTokens = resolveImageToolMaxTokens(model.maxTokens, params.maxTokens ?? 512); const completeImage = async (onPayload?: ProviderStreamOptions["onPayload"]) => { diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index 35d53be7996..3d185c2de65 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -3,6 +3,7 @@ import { join, relative, resolve } from "node:path"; import fg from "fast-glob"; import { describe, expect, it } from "vitest"; import { createNodeTestShards } from "../../scripts/lib/ci-node-test-plan.mjs"; +import { commandsLightTestFiles } from "../vitest/vitest.commands-light-paths.mjs"; import { createPluginsVitestConfig } from "../vitest/vitest.plugins.config.ts"; type VitestTestConfig = { @@ -169,10 +170,14 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { ]); }); - it("splits the agentic lane into control-plane, commands, agent, SDK, and plugin shards", () => { + it("splits the agentic lane into control-plane, command, agent, SDK, and plugin shards", () => { const shards = createNodeTestShards(); const controlPlaneShard = shards.find((shard) => shard.shardName === "agentic-control-plane"); - const commandsShard = shards.find((shard) => shard.shardName === "agentic-commands"); + const cliShard = shards.find((shard) => shard.shardName === "agentic-cli"); + const commandSupportShard = shards.find( + (shard) => shard.shardName === "agentic-command-support", + ); + const commandShards = shards.filter((shard) => shard.shardName.startsWith("agentic-commands-")); const agentShard = shards.find((shard) => shard.shardName === "agentic-agents"); const pluginSdkShard = shards.find((shard) => shard.shardName === "agentic-plugin-sdk"); const pluginsShard = shards.find((shard) => shard.shardName === "agentic-plugins"); @@ -184,17 +189,46 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { runner: "blacksmith-4vcpu-ubuntu-2404", requiresDist: false, }); - expect(commandsShard).toEqual({ - checkName: "checks-node-agentic-commands", - shardName: "agentic-commands", + expect(cliShard).toEqual({ + checkName: "checks-node-agentic-cli", + shardName: "agentic-cli", + configs: ["test/vitest/vitest.cli.config.ts"], + requiresDist: false, + }); + expect(commandSupportShard).toEqual({ + checkName: "checks-node-agentic-command-support", + shardName: "agentic-command-support", configs: [ - "test/vitest/vitest.cli.config.ts", "test/vitest/vitest.commands-light.config.ts", - "test/vitest/vitest.commands.config.ts", "test/vitest/vitest.daemon.config.ts", ], requiresDist: false, }); + expect(commandShards.map((shard) => shard.shardName)).toEqual([ + "agentic-commands-agent-channel", + "agentic-commands-doctor", + "agentic-commands-doctor-shared", + "agentic-commands-models", + "agentic-commands-onboard-config", + "agentic-commands-status-tools", + ]); + expect(commandShards).toEqual( + commandShards.map((shard) => ({ + checkName: `checks-node-${shard.shardName}`, + configs: ["test/vitest/vitest.commands.config.ts"], + includePatterns: shard.includePatterns, + requiresDist: false, + shardName: shard.shardName, + })), + ); + const commandShardFiles = commandShards + .flatMap((shard) => shard.includePatterns ?? []) + .toSorted((a, b) => a.localeCompare(b)); + const expectedCommandFiles = listTestFiles("src/commands") + .filter((file) => !commandsLightTestFiles.includes(file)) + .toSorted((a, b) => a.localeCompare(b)); + expect(commandShardFiles).toEqual(expectedCommandFiles); + expect(new Set(commandShardFiles).size).toBe(commandShardFiles.length); expect(agentShard).toEqual({ checkName: "checks-node-agentic-agents", shardName: "agentic-agents", diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index fade18bd645..2b120a873cc 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -283,7 +283,15 @@ describe("package artifact reuse", () => { 'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"', ); expect(releaseWorkflow).toContain("matrix_args+=(--fail-fast)"); - expect(releaseWorkflow).toContain('pnpm openclaw qa matrix "${matrix_args[@]}"'); + expect(releaseWorkflow).toContain( + 'pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"', + ); + expect(releaseWorkflow).toContain( + 'echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2', + ); + expect(releaseWorkflow).toContain( + 'echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2', + ); expect(qaWorkflow).toContain( 'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"', ); diff --git a/test/vitest/vitest.commands.config.ts b/test/vitest/vitest.commands.config.ts index a6cc50557a1..6f6041450cb 100644 --- a/test/vitest/vitest.commands.config.ts +++ b/test/vitest/vitest.commands.config.ts @@ -7,6 +7,7 @@ export function createCommandsVitestConfig(env?: Record