diff --git a/extensions/qa-matrix/src/cli.test.ts b/extensions/qa-matrix/src/cli.test.ts index 4a5dd704f6a..31668e22091 100644 --- a/extensions/qa-matrix/src/cli.test.ts +++ b/extensions/qa-matrix/src/cli.test.ts @@ -1,8 +1,55 @@ import { Command } from "commander"; -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { runQaMatrixCommand } = vi.hoisted(() => ({ + runQaMatrixCommand: vi.fn(), +})); + +vi.mock("./cli.runtime.js", () => ({ + runQaMatrixCommand, +})); + import { matrixQaCliRegistration } from "./cli.js"; +function mockProcessWrite( + _chunk: string | Uint8Array, + encodingOrCallback?: BufferEncoding | ((err?: Error | null) => void), + callback?: (err?: Error | null) => void, +) { + if (typeof encodingOrCallback === "function") { + encodingOrCallback(); + } else { + callback?.(); + } + return true; +} + describe("matrix qa cli registration", () => { + const originalDisableForceExit = process.env.OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT; + let exitSpy: ReturnType; + let stderrSpy: ReturnType; + let stdoutSpy: ReturnType; + + beforeEach(() => { + runQaMatrixCommand.mockReset(); + exitSpy = vi.spyOn(process, "exit").mockImplementation((code?: string | number | null) => { + throw new Error(`process.exit(${String(code)})`); + }); + stderrSpy = vi.spyOn(process.stderr, "write").mockImplementation(mockProcessWrite); + stdoutSpy = vi.spyOn(process.stdout, "write").mockImplementation(mockProcessWrite); + }); + + afterEach(() => { + if (originalDisableForceExit === undefined) { + delete process.env.OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT; + } else { + process.env.OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT = originalDisableForceExit; + } + exitSpy.mockRestore(); + stderrSpy.mockRestore(); + stdoutSpy.mockRestore(); + }); + it("keeps disposable Matrix lane flags focused", () => { const qa = new Command(); @@ -26,4 +73,27 @@ describe("matrix qa cli registration", () => { expect(optionNames).not.toContain("--credential-source"); expect(optionNames).not.toContain("--credential-role"); }); + + it("exits with failure after Matrix artifacts are written for a failed run", async () => { + const qa = new Command(); + matrixQaCliRegistration.register(qa); + runQaMatrixCommand.mockRejectedValue(new Error("Matrix QA failed.\nreport: /tmp/report.md")); + + await expect(qa.parseAsync(["node", "openclaw", "matrix"])).rejects.toThrow("process.exit(1)"); + + expect(runQaMatrixCommand).toHaveBeenCalledOnce(); + expect(stderrSpy).toHaveBeenCalledWith("Matrix QA failed.\nreport: /tmp/report.md\n"); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + + it("can disable the forced exit for direct test harnesses", async () => { + process.env.OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT = "1"; + const qa = new Command(); + matrixQaCliRegistration.register(qa); + runQaMatrixCommand.mockRejectedValue(new Error("scenario failed")); + + await expect(qa.parseAsync(["node", "openclaw", "matrix"])).rejects.toThrow("scenario failed"); + + expect(exitSpy).not.toHaveBeenCalled(); + }); }); diff --git a/extensions/qa-matrix/src/cli.ts b/extensions/qa-matrix/src/cli.ts index f95776850f7..8e5018c2e02 100644 --- a/extensions/qa-matrix/src/cli.ts +++ b/extensions/qa-matrix/src/cli.ts @@ -1,4 +1,5 @@ import type { Command } from "commander"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { createLazyCliRuntimeLoader, createLiveTransportQaCliRegistration, @@ -8,6 +9,8 @@ import { type MatrixQaCliRuntime = typeof import("./cli.runtime.js"); +const DISABLE_MATRIX_QA_FORCE_EXIT_ENV = "OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT"; + const loadMatrixQaCliRuntime = createLazyCliRuntimeLoader( () => import("./cli.runtime.js"), ); @@ -25,16 +28,26 @@ async function flushProcessStream(stream: NodeJS.WriteStream) { }); } +async function exitMatrixQaCommand(code: number): Promise { + // Matrix crypto native handles can outlive the QA run even after every + // client/gateway/harness has been stopped. This command is single-shot, so + // artifact completion should terminate deterministically on both pass and fail. + await Promise.all([flushProcessStream(process.stdout), flushProcessStream(process.stderr)]); + process.exit(code); +} + async function runQaMatrix(opts: LiveTransportQaCommandOptions) { const runtime = await loadMatrixQaCliRuntime(); - await runtime.runQaMatrixCommand(opts); - if (process.env.OPENCLAW_QA_MATRIX_DISABLE_SUCCESS_EXIT !== "1") { - // Matrix crypto native handles can outlive the QA run even after every - // client/gateway/harness has been stopped. This command is single-shot, so - // a successful artifact write should terminate deterministically instead of - // waiting for external timeout cleanup. - await Promise.all([flushProcessStream(process.stdout), flushProcessStream(process.stderr)]); - process.exit(0); + if (process.env[DISABLE_MATRIX_QA_FORCE_EXIT_ENV] === "1") { + await runtime.runQaMatrixCommand(opts); + return; + } + try { + await runtime.runQaMatrixCommand(opts); + await exitMatrixQaCommand(0); + } catch (error) { + process.stderr.write(`${formatErrorMessage(error)}\n`); + await exitMatrixQaCommand(1); } }