test: stabilize release validation test harnesses

2026-05-27 23:12:52 +00:00 · 2026-05-25 21:35:22 +01:00
parent f1197ed6fc
commit 5a9673ecd7
6 changed files with 52 additions and 12 deletions
--- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml
+++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml
@@ -2278,7 +2278,7 @@ jobs:
        include:
          - suite_id: live-gateway-docker
            label: Docker live gateway OpenAI
-            command: OPENCLAW_LIVE_GATEWAY_PROVIDERS=openai OPENCLAW_LIVE_GATEWAY_MODELS=openai/gpt-5.5 OPENCLAW_LIVE_GATEWAY_MAX_MODELS=1 OPENCLAW_LIVE_GATEWAY_STEP_TIMEOUT_MS=90000 OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS=300000 OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" timeout --foreground --kill-after=30s 35m bash .release-harness/scripts/test-live-gateway-models-docker.sh
+            command: OPENCLAW_LIVE_GATEWAY_THINKING=low OPENCLAW_LIVE_GATEWAY_PROVIDERS=openai OPENCLAW_LIVE_GATEWAY_MODELS=openai/gpt-5.5 OPENCLAW_LIVE_GATEWAY_MAX_MODELS=1 OPENCLAW_LIVE_GATEWAY_STEP_TIMEOUT_MS=90000 OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS=600000 OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" timeout --foreground --kill-after=30s 35m bash .release-harness/scripts/test-live-gateway-models-docker.sh
            timeout_minutes: 40
            profile_env_only: false
            profiles: beta minimum stable full
--- a/extensions/discord/src/voice/manager.e2e.test.ts
+++ b/extensions/discord/src/voice/manager.e2e.test.ts
@@ -1912,7 +1912,11 @@ describe("DiscordVoiceManager", () => {

    await manager.join({ guildId: "g1", channelId: "1001" });

-    expect(entersStateMock).toHaveBeenCalledWith(connection, "ready", 30_000);
+    const readyCall = entersStateMock.mock.calls[0];
+    expect(readyCall?.[0]).toBe(connection);
+    expect(readyCall?.[1]).toBe("ready");
+    expect(readyCall?.[2]).toBeGreaterThanOrEqual(29_900);
+    expect(readyCall?.[2]).toBeLessThanOrEqual(30_000);
  });

  it("deduplicates concurrent joins for the same guild and channel", async () => {
--- a/scripts/e2e/mcp-channels-docker-client.ts
+++ b/scripts/e2e/mcp-channels-docker-client.ts
@@ -341,13 +341,34 @@ async function main() {
      message: "yes abcde",
      idempotencyKey: randomUUID(),
    });
-    const permission = await waitFor(
-      "Claude permission notification",
-      () =>
-        mcpHandle.rawMessages
-          .map((entry) => ClaudePermissionNotificationSchema.safeParse(entry))
-          .find((entry) => entry.success && entry.data.params.request_id === "abcde")?.data.params,
-    );
+    let permission: { request_id: string; behavior: "allow" | "deny" };
+    try {
+      permission = await waitFor(
+        "Claude permission notification",
+        () =>
+          mcpHandle.rawMessages
+            .map((entry) => ClaudePermissionNotificationSchema.safeParse(entry))
+            .find((entry) => entry.success && entry.data.params.request_id === "abcde")?.data
+            .params,
+        60_000,
+      );
+    } catch (error) {
+      throw new Error(
+        `timeout waiting for Claude permission notification: ${JSON.stringify(
+          {
+            rawMessages: mcpHandle.rawMessages.slice(-10),
+            recentGatewayEvents: gateway.events.slice(-10).map((entry) => ({
+              event: entry.event,
+              sessionKey: entry.payload.sessionKey,
+              text: extractTextFromGatewayPayload(entry.payload),
+            })),
+          },
+          null,
+          2,
+        )}`,
+        { cause: error },
+      );
+    }
    assert(permission.behavior === "allow", "expected allow permission reply");

    process.stdout.write(
--- a/scripts/lib/live-docker-auth.sh
+++ b/scripts/lib/live-docker-auth.sh
@@ -93,7 +93,7 @@ openclaw_live_should_include_auth_file_for_provider() {
  local provider
  provider="$(openclaw_live_trim "${1:-}")"
  case "$provider" in
-    codex-cli | openai-codex)
+    codex-cli | openai | openai-codex)
      printf '%s\n' ".codex/auth.json"
      printf '%s\n' ".codex/config.toml"
      ;;
--- a/scripts/test-live-gateway-models-docker.sh
+++ b/scripts/test-live-gateway-models-docker.sh
@@ -175,6 +175,18 @@ echo "==> External auth files: ${AUTH_FILES_CSV:-none}"
 DOCKER_RUN_ARGS=(docker run --rm -t \
  -u "$DOCKER_USER" \
  --entrypoint bash \
+  -e OPENAI_API_KEY \
+  -e OPENAI_BASE_URL \
+  -e ANTHROPIC_API_KEY \
+  -e GEMINI_API_KEY \
+  -e GOOGLE_API_KEY \
+  -e MINIMAX_API_KEY \
+  -e OPENROUTER_API_KEY \
+  -e FIREWORKS_API_KEY \
+  -e DEEPSEEK_API_KEY \
+  -e XAI_API_KEY \
+  -e ZAI_API_KEY \
+  -e Z_AI_API_KEY \
  -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
  -e HOME=/home/node \
  -e NODE_OPTIONS="$CONTAINER_NODE_OPTIONS" \
@@ -191,6 +203,7 @@ DOCKER_RUN_ARGS=(docker run --rm -t \
  -e OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS="${OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS:-}" \
  -e OPENCLAW_LIVE_GATEWAY_MODELS="${OPENCLAW_LIVE_GATEWAY_MODELS:-modern}" \
  -e OPENCLAW_LIVE_GATEWAY_PROVIDERS="${OPENCLAW_LIVE_GATEWAY_PROVIDERS:-}" \
+  -e OPENCLAW_LIVE_GATEWAY_THINKING="${OPENCLAW_LIVE_GATEWAY_THINKING:-}" \
  -e OPENCLAW_LIVE_GATEWAY_SMOKE="${OPENCLAW_LIVE_GATEWAY_SMOKE:-1}" \
  -e OPENCLAW_LIVE_GATEWAY_MAX_MODELS="${OPENCLAW_LIVE_GATEWAY_MAX_MODELS:-8}" \
  -e OPENCLAW_LIVE_GATEWAY_HEARTBEAT_MS="${OPENCLAW_LIVE_GATEWAY_HEARTBEAT_MS:-}" \
--- a/test/scripts/package-acceptance-workflow.test.ts
+++ b/test/scripts/package-acceptance-workflow.test.ts
@@ -525,7 +525,7 @@ describe("package artifact reuse", () => {
    expect(workflow).toContain("suite_id: live-gateway-anthropic-docker");
    expect(workflow).toContain("OPENCLAW_LIVE_GATEWAY_MAX_MODELS=2");
    expect(workflow).toContain(
-      "OPENCLAW_LIVE_GATEWAY_PROVIDERS=openai OPENCLAW_LIVE_GATEWAY_MODELS=openai/gpt-5.5 OPENCLAW_LIVE_GATEWAY_MAX_MODELS=1",
+      "OPENCLAW_LIVE_GATEWAY_THINKING=low OPENCLAW_LIVE_GATEWAY_PROVIDERS=openai OPENCLAW_LIVE_GATEWAY_MODELS=openai/gpt-5.5 OPENCLAW_LIVE_GATEWAY_MAX_MODELS=1 OPENCLAW_LIVE_GATEWAY_STEP_TIMEOUT_MS=90000 OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS=600000",
    );
    expect(workflow).toContain("timeout --foreground --kill-after=30s 35m");
    expect(workflow).toMatch(/suite_id: live-gateway-docker[\s\S]*?timeout_minutes: 40/u);
@@ -582,6 +582,7 @@ describe("package artifact reuse", () => {
    const scenarios = readFileSync("scripts/lib/docker-e2e-scenarios.mjs", "utf8");
    const scheduler = readFileSync("scripts/test-docker-all.mjs", "utf8");
    const harness = readFileSync("scripts/test-live-codex-harness-docker.sh", "utf8");
+    const liveDockerAuth = readFileSync("scripts/lib/live-docker-auth.sh", "utf8");
    const sharedLiveScripts = [
      readFileSync("scripts/test-live-models-docker.sh", "utf8"),
      readFileSync("scripts/test-live-gateway-models-docker.sh", "utf8"),
@@ -596,7 +597,7 @@ describe("package artifact reuse", () => {
      'run: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" timeout --foreground --kill-after=30s 35m bash .release-harness/scripts/test-live-models-docker.sh',
    );
    expect(workflow).toContain(
-      "command: OPENCLAW_LIVE_GATEWAY_PROVIDERS=openai OPENCLAW_LIVE_GATEWAY_MODELS=openai/gpt-5.5 OPENCLAW_LIVE_GATEWAY_MAX_MODELS=1",
+      "command: OPENCLAW_LIVE_GATEWAY_THINKING=low OPENCLAW_LIVE_GATEWAY_PROVIDERS=openai OPENCLAW_LIVE_GATEWAY_MODELS=openai/gpt-5.5 OPENCLAW_LIVE_GATEWAY_MAX_MODELS=1",
    );
    expect(workflow).toContain(
      'command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" timeout --foreground --kill-after=30s 45m bash .release-harness/scripts/test-live-cli-backend-docker.sh',
@@ -627,6 +628,7 @@ describe("package artifact reuse", () => {
    );
    expect(scheduler).toContain("function liveDockerHarnessScriptCommand");
    expect(scheduler).toContain('liveDockerHarnessScriptCommand("test-live-build-docker.sh")');
+    expect(liveDockerAuth).toContain("codex-cli | openai | openai-codex)");
    expect(harness).toContain('source "$TRUSTED_HARNESS_DIR/scripts/lib/live-docker-auth.sh"');
    expect(harness).not.toContain('source "$ROOT_DIR/scripts/lib/live-docker-auth.sh"');
    expect(harness).toContain(