ci: shard release docker plugin validation

.github/workflows/openclaw-live-and-e2e-checks-reusable.yml

@@ -475,6 +475,18 @@ jobs:
           - chunk_id: plugins-runtime-install-d
             label: plugins/runtime install D
             timeout_minutes: 120
+          - chunk_id: plugins-runtime-install-e
+            label: plugins/runtime install E
+            timeout_minutes: 120
+          - chunk_id: plugins-runtime-install-f
+            label: plugins/runtime install F
+            timeout_minutes: 120
+          - chunk_id: plugins-runtime-install-g
+            label: plugins/runtime install G
+            timeout_minutes: 120
+          - chunk_id: plugins-runtime-install-h
+            label: plugins/runtime install H
+            timeout_minutes: 120
           - chunk_id: bundled-channels-core
             label: bundled channels core
             timeout_minutes: 90

docs/reference/RELEASING.md

@@ -349,16 +349,21 @@ Release Docker coverage includes:
 - full install smoke with the slow Bun global install smoke enabled
 - repository E2E lanes
 - release-path Docker chunks: `core`, `package-update-openai`,
-  `package-update-anthropic`, `package-update-core`, `plugins-runtime-core`,
+  `package-update-anthropic`, `package-update-core`, `plugins-runtime-plugins`,
+  `plugins-runtime-services`,
   `plugins-runtime-install-a`, `plugins-runtime-install-b`,
+  `plugins-runtime-install-c`, `plugins-runtime-install-d`,
+  `plugins-runtime-install-e`, `plugins-runtime-install-f`,
+  `plugins-runtime-install-g`, `plugins-runtime-install-h`,
   `bundled-channels-core`, `bundled-channels-update-a`,
-  `bundled-channels-update-b`, and `bundled-channels-contracts`
-- OpenWebUI coverage inside the `plugins-runtime-core` chunk when requested
+  `bundled-channels-update-discord`, `bundled-channels-update-b`, and
+  `bundled-channels-contracts`
+- OpenWebUI coverage inside the `plugins-runtime-services` chunk when requested
 - split bundled-channel dependency lanes across channel-smoke, update-target,
   and setup/runtime contract chunks instead of one large bundled-channel job
 - split bundled plugin install/uninstall lanes
   `bundled-plugin-install-uninstall-0` through
-  `bundled-plugin-install-uninstall-7`
+  `bundled-plugin-install-uninstall-23`
 - live/E2E provider suites and Docker live model coverage when release checks
   include live suites
 

scripts/ci-docker-pull-retry.sh

@@ -8,8 +8,8 @@ fi
 
 image="$1"
 attempts="${OPENCLAW_DOCKER_PULL_ATTEMPTS:-3}"
-timeout_seconds="${OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS:-480}"
-retry_delay_seconds="${OPENCLAW_DOCKER_PULL_RETRY_DELAY_SECONDS:-20}"
+timeout_seconds="${OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS:-180}"
+retry_delay_seconds="${OPENCLAW_DOCKER_PULL_RETRY_DELAY_SECONDS:-5}"
 
 if ! [[ "$attempts" =~ ^[1-9][0-9]*$ ]]; then
   echo "OPENCLAW_DOCKER_PULL_ATTEMPTS must be a positive integer, got: $attempts" >&2

scripts/e2e/lib/bundled-plugin-install-uninstall/sweep.sh

@@ -32,11 +32,13 @@ for plugin_entry in "${plugin_entries[@]}"; do
   IFS=$'\t' read -r plugin_id plugin_dir requires_config <<<"$plugin_entry"
   install_log="/tmp/openclaw-install-${plugin_index}.log"
   uninstall_log="/tmp/openclaw-uninstall-${plugin_index}.log"
+  plugin_started_at="$(date +%s)"
   echo "Installing bundled plugin: $plugin_id ($plugin_dir)"
   node "$OPENCLAW_ENTRY" plugins install "$plugin_id" >"$install_log" 2>&1 || {
     cat "$install_log"
     exit 1
   }
+  install_finished_at="$(date +%s)"
   node "$probe" assert-installed "$plugin_id" "$plugin_dir" "$requires_config"
 
   echo "Uninstalling bundled plugin: $plugin_id ($plugin_dir)"
@@ -44,7 +46,9 @@ for plugin_entry in "${plugin_entries[@]}"; do
     cat "$uninstall_log"
     exit 1
   }
+  uninstall_finished_at="$(date +%s)"
   node "$probe" assert-uninstalled "$plugin_id" "$plugin_dir"
+  echo "Bundled plugin lifecycle timing: $plugin_id install=$((install_finished_at - plugin_started_at))s uninstall=$((uninstall_finished_at - install_finished_at))s total=$((uninstall_finished_at - plugin_started_at))s"
   plugin_index=$((plugin_index + 1))
 done
 

scripts/lib/docker-e2e-scenarios.mjs

@@ -9,7 +9,7 @@ const LIVE_ACP_TIMEOUT_MS = 20 * 60 * 1000;
 const LIVE_CLI_TIMEOUT_MS = 20 * 60 * 1000;
 const LIVE_PROFILE_TIMEOUT_MS = 20 * 60 * 1000;
 const OPENWEBUI_TIMEOUT_MS = 20 * 60 * 1000;
-export const BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS = 8;
+export const BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS = 24;
 
 export const LIVE_RETRY_PATTERNS = [
   /529\b/i,
@@ -180,7 +180,7 @@ const bundledPluginInstallUninstallLanes = Array.from(
       `bundled-plugin-install-uninstall-${index}`,
       `OPENCLAW_BUNDLED_PLUGIN_SWEEP_TOTAL=${BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS} OPENCLAW_BUNDLED_PLUGIN_SWEEP_INDEX=${index} OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:bundled-plugin-install-uninstall`,
       {
-        estimateSeconds: 280,
+        estimateSeconds: 120,
         resources: ["npm"],
         stateScenario: "empty",
         weight: 1,
@@ -563,10 +563,14 @@ const primaryReleasePathChunks = {
   "package-update-core": releasePathPackageUpdateCoreLanes,
   "plugins-runtime-plugins": releasePathPluginRuntimePluginLanes,
   "plugins-runtime-services": releasePathPluginRuntimeServiceLanes,
-  "plugins-runtime-install-a": bundledPluginInstallUninstallLanes.slice(0, 2),
-  "plugins-runtime-install-b": bundledPluginInstallUninstallLanes.slice(2, 4),
-  "plugins-runtime-install-c": bundledPluginInstallUninstallLanes.slice(4, 6),
-  "plugins-runtime-install-d": bundledPluginInstallUninstallLanes.slice(6),
+  "plugins-runtime-install-a": bundledPluginInstallUninstallLanes.slice(0, 3),
+  "plugins-runtime-install-b": bundledPluginInstallUninstallLanes.slice(3, 6),
+  "plugins-runtime-install-c": bundledPluginInstallUninstallLanes.slice(6, 9),
+  "plugins-runtime-install-d": bundledPluginInstallUninstallLanes.slice(9, 12),
+  "plugins-runtime-install-e": bundledPluginInstallUninstallLanes.slice(12, 15),
+  "plugins-runtime-install-f": bundledPluginInstallUninstallLanes.slice(15, 18),
+  "plugins-runtime-install-g": bundledPluginInstallUninstallLanes.slice(18, 21),
+  "plugins-runtime-install-h": bundledPluginInstallUninstallLanes.slice(21),
   "bundled-channels-core": [releasePathBundledChannelLanes[0], ...bundledChannelSmokeLanes],
   "bundled-channels-update-a": [bundledChannelUpdateLanes[0], bundledChannelUpdateLanes[4]],
   "bundled-channels-update-discord": [bundledChannelUpdateLanes[1]],

test/scripts/docker-e2e-plan.test.ts

@@ -5,6 +5,7 @@ import {
   parseLaneSelection,
   resolveDockerE2ePlan,
 } from "../../scripts/lib/docker-e2e-plan.mjs";
+import { BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS } from "../../scripts/lib/docker-e2e-scenarios.mjs";
 
 const orderLanes = <T>(lanes: T[]) => lanes;
 
@@ -47,7 +48,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
     expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-channel-feishu");
     expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-channel-update-acpx");
     expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-plugin-install-uninstall-0");
-    expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-plugin-install-uninstall-7");
+    expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-plugin-install-uninstall-23");
     expect(plan.lanes.filter((lane) => lane.name === "install-e2e-openai")).toHaveLength(1);
     expect(
       plan.lanes.filter((lane) => lane.name === "bundled-plugin-install-uninstall-0"),
@@ -119,6 +120,26 @@ describe("scripts/lib/docker-e2e-plan", () => {
       profile: RELEASE_PATH_PROFILE,
       releaseChunk: "plugins-runtime-install-d",
     });
+    const pluginsRuntimeInstallE = planFor({
+      includeOpenWebUI: true,
+      profile: RELEASE_PATH_PROFILE,
+      releaseChunk: "plugins-runtime-install-e",
+    });
+    const pluginsRuntimeInstallF = planFor({
+      includeOpenWebUI: true,
+      profile: RELEASE_PATH_PROFILE,
+      releaseChunk: "plugins-runtime-install-f",
+    });
+    const pluginsRuntimeInstallG = planFor({
+      includeOpenWebUI: true,
+      profile: RELEASE_PATH_PROFILE,
+      releaseChunk: "plugins-runtime-install-g",
+    });
+    const pluginsRuntimeInstallH = planFor({
+      includeOpenWebUI: true,
+      profile: RELEASE_PATH_PROFILE,
+      releaseChunk: "plugins-runtime-install-h",
+    });
     const bundledChannelsCore = planFor({
       includeOpenWebUI: true,
       profile: RELEASE_PATH_PROFILE,
@@ -190,18 +211,42 @@ describe("scripts/lib/docker-e2e-plan", () => {
     expect(pluginsRuntimeInstallA.lanes.map((lane) => lane.name)).toEqual([
       "bundled-plugin-install-uninstall-0",
       "bundled-plugin-install-uninstall-1",
+      "bundled-plugin-install-uninstall-2",
     ]);
     expect(pluginsRuntimeInstallB.lanes.map((lane) => lane.name)).toEqual([
-      "bundled-plugin-install-uninstall-2",
       "bundled-plugin-install-uninstall-3",
-    ]);
-    expect(pluginsRuntimeInstallC.lanes.map((lane) => lane.name)).toEqual([
       "bundled-plugin-install-uninstall-4",
       "bundled-plugin-install-uninstall-5",
     ]);
-    expect(pluginsRuntimeInstallD.lanes.map((lane) => lane.name)).toEqual([
+    expect(pluginsRuntimeInstallC.lanes.map((lane) => lane.name)).toEqual([
       "bundled-plugin-install-uninstall-6",
       "bundled-plugin-install-uninstall-7",
+      "bundled-plugin-install-uninstall-8",
+    ]);
+    expect(pluginsRuntimeInstallD.lanes.map((lane) => lane.name)).toEqual([
+      "bundled-plugin-install-uninstall-9",
+      "bundled-plugin-install-uninstall-10",
+      "bundled-plugin-install-uninstall-11",
+    ]);
+    expect(pluginsRuntimeInstallE.lanes.map((lane) => lane.name)).toEqual([
+      "bundled-plugin-install-uninstall-12",
+      "bundled-plugin-install-uninstall-13",
+      "bundled-plugin-install-uninstall-14",
+    ]);
+    expect(pluginsRuntimeInstallF.lanes.map((lane) => lane.name)).toEqual([
+      "bundled-plugin-install-uninstall-15",
+      "bundled-plugin-install-uninstall-16",
+      "bundled-plugin-install-uninstall-17",
+    ]);
+    expect(pluginsRuntimeInstallG.lanes.map((lane) => lane.name)).toEqual([
+      "bundled-plugin-install-uninstall-18",
+      "bundled-plugin-install-uninstall-19",
+      "bundled-plugin-install-uninstall-20",
+    ]);
+    expect(pluginsRuntimeInstallH.lanes.map((lane) => lane.name)).toEqual([
+      "bundled-plugin-install-uninstall-21",
+      "bundled-plugin-install-uninstall-22",
+      "bundled-plugin-install-uninstall-23",
     ]);
     expect(bundledChannelsCore.lanes.map((lane) => lane.name)).toEqual([
       "plugin-update",
@@ -274,7 +319,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
       expect.arrayContaining([
         "plugins",
         "bundled-plugin-install-uninstall-0",
-        "bundled-plugin-install-uninstall-7",
+        "bundled-plugin-install-uninstall-23",
         "openwebui",
       ]),
     );
@@ -468,9 +513,12 @@ describe("scripts/lib/docker-e2e-plan", () => {
     const plan = planFor({ selectedLaneNames });
 
     expect(selectedLaneNames).toEqual(
-      Array.from({ length: 8 }, (_, index) => `bundled-plugin-install-uninstall-${index}`),
+      Array.from(
+        { length: BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS },
+        (_, index) => `bundled-plugin-install-uninstall-${index}`,
+      ),
     );
-    expect(plan.lanes).toHaveLength(8);
+    expect(plan.lanes).toHaveLength(BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS);
     expect(plan.lanes[0]).toEqual(
       expect.objectContaining({
         command: expect.stringContaining("OPENCLAW_BUNDLED_PLUGIN_SWEEP_INDEX=0"),
@@ -480,12 +528,12 @@ describe("scripts/lib/docker-e2e-plan", () => {
         resources: expect.arrayContaining(["docker", "npm"]),
       }),
     );
-    expect(plan.lanes[7]).toEqual(
+    expect(plan.lanes[23]).toEqual(
       expect.objectContaining({
-        command: expect.stringContaining("OPENCLAW_BUNDLED_PLUGIN_SWEEP_INDEX=7"),
+        command: expect.stringContaining("OPENCLAW_BUNDLED_PLUGIN_SWEEP_INDEX=23"),
         imageKind: "functional",
         live: false,
-        name: "bundled-plugin-install-uninstall-7",
+        name: "bundled-plugin-install-uninstall-23",
         resources: expect.arrayContaining(["docker", "npm"]),
       }),
     );

test/scripts/package-acceptance-workflow.test.ts

@@ -99,6 +99,10 @@ describe("package artifact reuse", () => {
 
     expect(pullHelper).toContain("OPENCLAW_DOCKER_PULL_ATTEMPTS");
     expect(pullHelper).toContain("OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS");
+    expect(pullHelper).toContain('timeout_seconds="${OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS:-180}"');
+    expect(pullHelper).toContain(
+      'retry_delay_seconds="${OPENCLAW_DOCKER_PULL_RETRY_DELAY_SECONDS:-5}"',
+    );
     expect(pullHelper).toContain(
       'timeout --foreground --kill-after=30s "${timeout_seconds}s" docker pull "$image"',
     );

test/scripts/plugin-prerelease-test-plan.test.ts

@@ -3,6 +3,7 @@ import { readFileSync } from "node:fs";
 import { describe, expect, it } from "vitest";
 import { parse } from "yaml";
 import { findLaneByName } from "../../scripts/lib/docker-e2e-plan.mjs";
+import { BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS } from "../../scripts/lib/docker-e2e-scenarios.mjs";
 import {
   PLUGIN_PRERELEASE_REQUIRED_SURFACES,
   assertPluginPrereleaseTestPlanComplete,
@@ -46,14 +47,10 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => {
       "gateway-network",
       "mcp-channels",
       "cron-mcp-cleanup",
-      "bundled-plugin-install-uninstall-0",
-      "bundled-plugin-install-uninstall-1",
-      "bundled-plugin-install-uninstall-2",
-      "bundled-plugin-install-uninstall-3",
-      "bundled-plugin-install-uninstall-4",
-      "bundled-plugin-install-uninstall-5",
-      "bundled-plugin-install-uninstall-6",
-      "bundled-plugin-install-uninstall-7",
+      ...Array.from(
+        { length: BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS },
+        (_, index) => `bundled-plugin-install-uninstall-${index}`,
+      ),
     ]);
 
     for (const lane of plan.dockerLanes) {