ci(test): split command shards and harden release checks

.github/workflows/openclaw-release-checks.yml

@@ -665,7 +665,6 @@ jobs:
 
           matrix_args=(
             --repo-root . \
-            --output-dir "${output_dir}" \
             --provider-mode mock-openai \
             --model mock-openai/gpt-5.5 \
             --alt-model mock-openai/gpt-5.5-alt \
@@ -676,7 +675,17 @@ jobs:
             matrix_args+=(--fail-fast)
           fi
 
-          pnpm openclaw qa matrix "${matrix_args[@]}"
+          for attempt in 1 2; do
+            attempt_output_dir="${output_dir}/attempt-${attempt}"
+            if pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"; then
+              exit 0
+            fi
+            if [[ "${attempt}" == "2" ]]; then
+              exit 1
+            fi
+            echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2
+            sleep 10
+          done
 
       - name: Upload Matrix QA artifacts
         if: always()
@@ -751,15 +760,25 @@ jobs:
           output_dir=".artifacts/qa-e2e/telegram-live-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
           echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
 
-          pnpm openclaw qa telegram \
-            --repo-root . \
-            --output-dir "${output_dir}" \
-            --provider-mode mock-openai \
-            --model mock-openai/gpt-5.5 \
-            --alt-model mock-openai/gpt-5.5-alt \
-            --fast \
-            --credential-source convex \
-            --credential-role ci
+          for attempt in 1 2; do
+            attempt_output_dir="${output_dir}/attempt-${attempt}"
+            if pnpm openclaw qa telegram \
+              --repo-root . \
+              --output-dir "${attempt_output_dir}" \
+              --provider-mode mock-openai \
+              --model mock-openai/gpt-5.5 \
+              --alt-model mock-openai/gpt-5.5-alt \
+              --fast \
+              --credential-source convex \
+              --credential-role ci; then
+              exit 0
+            fi
+            if [[ "${attempt}" == "2" ]]; then
+              exit 1
+            fi
+            echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2
+            sleep 10
+          done
 
       - name: Upload Telegram QA artifacts
         if: always()

extensions/openai/openai.live.test.ts

@@ -13,6 +13,11 @@ import {
 } from "openclaw/plugin-sdk/plugin-test-runtime";
 import { runRealtimeSttLiveTest } from "openclaw/plugin-sdk/provider-test-contracts";
 import { getRuntimeConfig } from "openclaw/plugin-sdk/runtime-config-snapshot";
+import {
+  isOverloadedErrorMessage,
+  isServerErrorMessage,
+  isTimeoutErrorMessage,
+} from "openclaw/plugin-sdk/test-env";
 import { describe, expect, it } from "vitest";
 import plugin from "./index.js";
 
@@ -99,6 +104,21 @@ function createReferencePng(): Buffer {
   return encodePngRgba(buf, width, height);
 }
 
+function formatLiveOpenAIError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function resolveLiveOpenAISkipReason(error: unknown): string | null {
+  const message = formatLiveOpenAIError(error);
+  if (isTimeoutErrorMessage(message) || /timed out|operation was aborted/i.test(message)) {
+    return "provider timeout";
+  }
+  if (isOverloadedErrorMessage(message) || isServerErrorMessage(message)) {
+    return "provider outage";
+  }
+  return null;
+}
+
 function createLiveConfig(): OpenClawConfig {
   const cfg = getRuntimeConfig();
   return {
@@ -436,22 +456,36 @@ describeLive("openai plugin live", () => {
     const agentDir = await createTempAgentDir();
 
     try {
-      const description = await mediaProvider.describeImage?.({
-        buffer: createReferencePng(),
-        fileName: "reference.png",
-        mime: "image/png",
-        prompt: "Reply with one lowercase word for the dominant center color.",
-        timeoutMs: 120_000,
-        agentDir,
-        cfg,
-        authStore: EMPTY_AUTH_STORE,
-        model: LIVE_VISION_MODEL,
-        provider: "openai",
-      });
+      let description:
+        | Awaited<ReturnType<NonNullable<typeof mediaProvider.describeImage>>>
+        | undefined;
+      try {
+        description = await mediaProvider.describeImage?.({
+          buffer: createReferencePng(),
+          fileName: "reference.png",
+          mime: "image/png",
+          prompt: "Reply with one lowercase word for the dominant center color.",
+          timeoutMs: 45_000,
+          agentDir,
+          cfg,
+          authStore: EMPTY_AUTH_STORE,
+          model: LIVE_VISION_MODEL,
+          provider: "openai",
+        });
+      } catch (err) {
+        const skipReason = resolveLiveOpenAISkipReason(err);
+        if (skipReason) {
+          console.warn(
+            `[live:openai] image description skipped: ${skipReason}: ${formatLiveOpenAIError(err)}`,
+          );
+          return;
+        }
+        throw err;
+      }
 
       expect((description?.text ?? "").toLowerCase()).toContain("orange");
     } finally {
       await fs.rm(agentDir, { recursive: true, force: true });
     }
-  }, 180_000);
+  }, 75_000);
 });

scripts/lib/ci-node-test-plan.mjs

@@ -1,5 +1,6 @@
 import { existsSync, readdirSync } from "node:fs";
 import { join, relative } from "node:path";
+import { commandsLightTestFiles } from "../../test/vitest/vitest.commands-light-paths.mjs";
 import { fullSuiteVitestShards } from "../../test/vitest/vitest.test-shards.mjs";
 
 const EXCLUDED_FULL_SUITE_SHARDS = new Set([
@@ -84,6 +85,63 @@ function createAutoReplyReplySplitShards() {
     .filter((shard) => shard.includePatterns.length > 0);
 }
 
+function resolveCommandShardName(file) {
+  const name = relative("src/commands", file).replaceAll("\\", "/");
+  if (name.startsWith("agent") || name.startsWith("channel") || name === "message.test.ts") {
+    return "agentic-commands-agent-channel";
+  }
+  if (name.startsWith("doctor")) {
+    if (name.startsWith("doctor/shared/") || name.startsWith("doctor/")) {
+      return "agentic-commands-doctor-shared";
+    }
+    return "agentic-commands-doctor";
+  }
+  if (
+    name.startsWith("auth-choice") ||
+    name.startsWith("configure") ||
+    name.startsWith("onboard") ||
+    name === "setup.test.ts"
+  ) {
+    return "agentic-commands-onboard-config";
+  }
+  if (
+    name.startsWith("models/") ||
+    name === "model-picker.test.ts" ||
+    name === "openai-model-default.test.ts"
+  ) {
+    return "agentic-commands-models";
+  }
+  return "agentic-commands-status-tools";
+}
+
+function createAgenticCommandSplitShards() {
+  const commandsLightTests = new Set(commandsLightTestFiles);
+  const groups = new Map();
+  for (const file of listTestFiles("src/commands")) {
+    if (commandsLightTests.has(file)) {
+      continue;
+    }
+    const shardName = resolveCommandShardName(file);
+    groups.set(shardName, [...(groups.get(shardName) ?? []), file]);
+  }
+
+  return [
+    "agentic-commands-agent-channel",
+    "agentic-commands-doctor",
+    "agentic-commands-doctor-shared",
+    "agentic-commands-models",
+    "agentic-commands-onboard-config",
+    "agentic-commands-status-tools",
+  ]
+    .map((shardName) => ({
+      configs: ["test/vitest/vitest.commands.config.ts"],
+      includePatterns: groups.get(shardName) ?? [],
+      requiresDist: false,
+      shardName,
+    }))
+    .filter((shard) => shard.includePatterns.length > 0);
+}
+
 const SPLIT_NODE_SHARDS = new Map([
   [
     "core-unit-fast",
@@ -179,15 +237,19 @@ const SPLIT_NODE_SHARDS = new Map([
         runner: "blacksmith-4vcpu-ubuntu-2404",
       },
       {
-        shardName: "agentic-commands",
+        shardName: "agentic-cli",
+        configs: ["test/vitest/vitest.cli.config.ts"],
+        requiresDist: false,
+      },
+      {
+        shardName: "agentic-command-support",
         configs: [
-          "test/vitest/vitest.cli.config.ts",
           "test/vitest/vitest.commands-light.config.ts",
-          "test/vitest/vitest.commands.config.ts",
           "test/vitest/vitest.daemon.config.ts",
         ],
         requiresDist: false,
       },
+      ...createAgenticCommandSplitShards(),
       {
         shardName: "agentic-agents",
         configs: [

src/media-understanding/image.test.ts

@@ -553,6 +553,28 @@ describe("describeImageWithModel", () => {
     expect(options?.signal?.aborted).toBe(true);
   });
 
+  it("rejects when image runtime setup exceeds the request timeout", async () => {
+    vi.useFakeTimers();
+    ensureOpenClawModelsJsonMock.mockImplementationOnce(() => new Promise(() => {}));
+
+    const result = describeImageWithModel({
+      cfg: {},
+      agentDir: "/tmp/openclaw-agent",
+      provider: "openai",
+      model: "gpt-5.4-mini",
+      buffer: Buffer.from("png-bytes"),
+      fileName: "image.png",
+      mime: "image/png",
+      prompt: "Describe the image.",
+      timeoutMs: 25,
+    });
+
+    const assertion = expect(result).rejects.toThrow("image description timed out after 25ms");
+    await vi.advanceTimersByTimeAsync(25);
+    await assertion;
+    expect(completeMock).not.toHaveBeenCalled();
+  });
+
   it("normalizes deprecated google flash ids before lookup and keeps profile auth selection", async () => {
     const findMock = vi.fn((provider: string, modelId: string) => {
       expect(provider).toBe("google");

src/media-understanding/image.ts

@@ -349,11 +349,17 @@ async function describeImagesWithModelInternal(
   options: { onPayload?: ProviderStreamOptions["onPayload"] } = {},
 ): Promise<ImagesDescriptionResult> {
   const prompt = params.prompt ?? "Describe the image.";
+  const startedAtMs = Date.now();
+  const controller = new AbortController();
   let apiKey: string;
   let model: Model<Api> | undefined;
 
   try {
-    const resolved = await resolveImageRuntime(params);
+    const resolved = await withImageDescriptionTimeout({
+      controller,
+      timeoutMs: resolveImageDescriptionTimeoutMs(params.timeoutMs, startedAtMs),
+      task: resolveImageRuntime(params),
+    });
     apiKey = resolved.apiKey;
     model = resolved.model;
   } catch (err) {
@@ -391,8 +397,6 @@ async function describeImagesWithModelInternal(
   const context = buildImageContext(prompt, params.images, {
     promptInUserContent: shouldPlaceImagePromptInUserContent(model),
   });
-  const startedAtMs = Date.now();
-  const controller = new AbortController();
 
   const maxTokens = resolveImageToolMaxTokens(model.maxTokens, params.maxTokens ?? 512);
   const completeImage = async (onPayload?: ProviderStreamOptions["onPayload"]) => {

test/scripts/ci-node-test-plan.test.ts

@@ -3,6 +3,7 @@ import { join, relative, resolve } from "node:path";
 import fg from "fast-glob";
 import { describe, expect, it } from "vitest";
 import { createNodeTestShards } from "../../scripts/lib/ci-node-test-plan.mjs";
+import { commandsLightTestFiles } from "../vitest/vitest.commands-light-paths.mjs";
 import { createPluginsVitestConfig } from "../vitest/vitest.plugins.config.ts";
 
 type VitestTestConfig = {
@@ -169,10 +170,14 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
     ]);
   });
 
-  it("splits the agentic lane into control-plane, commands, agent, SDK, and plugin shards", () => {
+  it("splits the agentic lane into control-plane, command, agent, SDK, and plugin shards", () => {
     const shards = createNodeTestShards();
     const controlPlaneShard = shards.find((shard) => shard.shardName === "agentic-control-plane");
-    const commandsShard = shards.find((shard) => shard.shardName === "agentic-commands");
+    const cliShard = shards.find((shard) => shard.shardName === "agentic-cli");
+    const commandSupportShard = shards.find(
+      (shard) => shard.shardName === "agentic-command-support",
+    );
+    const commandShards = shards.filter((shard) => shard.shardName.startsWith("agentic-commands-"));
     const agentShard = shards.find((shard) => shard.shardName === "agentic-agents");
     const pluginSdkShard = shards.find((shard) => shard.shardName === "agentic-plugin-sdk");
     const pluginsShard = shards.find((shard) => shard.shardName === "agentic-plugins");
@@ -184,17 +189,46 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
       runner: "blacksmith-4vcpu-ubuntu-2404",
       requiresDist: false,
     });
-    expect(commandsShard).toEqual({
-      checkName: "checks-node-agentic-commands",
-      shardName: "agentic-commands",
+    expect(cliShard).toEqual({
+      checkName: "checks-node-agentic-cli",
+      shardName: "agentic-cli",
+      configs: ["test/vitest/vitest.cli.config.ts"],
+      requiresDist: false,
+    });
+    expect(commandSupportShard).toEqual({
+      checkName: "checks-node-agentic-command-support",
+      shardName: "agentic-command-support",
       configs: [
-        "test/vitest/vitest.cli.config.ts",
         "test/vitest/vitest.commands-light.config.ts",
-        "test/vitest/vitest.commands.config.ts",
         "test/vitest/vitest.daemon.config.ts",
       ],
       requiresDist: false,
     });
+    expect(commandShards.map((shard) => shard.shardName)).toEqual([
+      "agentic-commands-agent-channel",
+      "agentic-commands-doctor",
+      "agentic-commands-doctor-shared",
+      "agentic-commands-models",
+      "agentic-commands-onboard-config",
+      "agentic-commands-status-tools",
+    ]);
+    expect(commandShards).toEqual(
+      commandShards.map((shard) => ({
+        checkName: `checks-node-${shard.shardName}`,
+        configs: ["test/vitest/vitest.commands.config.ts"],
+        includePatterns: shard.includePatterns,
+        requiresDist: false,
+        shardName: shard.shardName,
+      })),
+    );
+    const commandShardFiles = commandShards
+      .flatMap((shard) => shard.includePatterns ?? [])
+      .toSorted((a, b) => a.localeCompare(b));
+    const expectedCommandFiles = listTestFiles("src/commands")
+      .filter((file) => !commandsLightTestFiles.includes(file))
+      .toSorted((a, b) => a.localeCompare(b));
+    expect(commandShardFiles).toEqual(expectedCommandFiles);
+    expect(new Set(commandShardFiles).size).toBe(commandShardFiles.length);
     expect(agentShard).toEqual({
       checkName: "checks-node-agentic-agents",
       shardName: "agentic-agents",

test/scripts/package-acceptance-workflow.test.ts

@@ -283,7 +283,15 @@ describe("package artifact reuse", () => {
       'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"',
     );
     expect(releaseWorkflow).toContain("matrix_args+=(--fail-fast)");
-    expect(releaseWorkflow).toContain('pnpm openclaw qa matrix "${matrix_args[@]}"');
+    expect(releaseWorkflow).toContain(
+      'pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"',
+    );
+    expect(releaseWorkflow).toContain(
+      'echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2',
+    );
+    expect(releaseWorkflow).toContain(
+      'echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2',
+    );
     expect(qaWorkflow).toContain(
       'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"',
     );

test/vitest/vitest.commands.config.ts

@@ -7,6 +7,7 @@ export function createCommandsVitestConfig(env?: Record<string, string | undefin
     env,
     exclude: commandsLightTestFiles,
     name: "commands",
+    pool: "forks",
   });
 }