diff --git a/extensions/qa-lab/src/crabline-transport.test.ts b/extensions/qa-lab/src/crabline-transport.test.ts index 7e1bdfd16529..1ddc71509c14 100644 --- a/extensions/qa-lab/src/crabline-transport.test.ts +++ b/extensions/qa-lab/src/crabline-transport.test.ts @@ -1,10 +1,7 @@ // Qa Lab tests cover Crabline local-provider transport integration behavior. import fs from "node:fs/promises"; import path from "node:path"; -import { - OPENCLAW_CRABLINE_MANIFEST_PATH, - type OpenClawCrablineChannelDriverSelection, -} from "@openclaw/crabline"; +import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; import { withTempDir } from "openclaw/plugin-sdk/test-env"; import { describe, expect, it } from "vitest"; @@ -20,6 +17,13 @@ function createSelection(channel: OpenClawCrablineChannelDriverSelection["channe } as const; } +function requireString(value: unknown, label: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`${label} is required`); + } + return value; +} + describe("crabline transport", () => { it("configures OpenClaw's Telegram plugin against a Crabline local provider server", async () => { await withTempDir("qa-crabline-transport-", async (outputDir) => { @@ -50,12 +54,9 @@ describe("crabline transport", () => { replyTo: "100001", }); - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - provider?: string; - }; - expect(manifest.provider).toBe("telegram"); + await expect( + fs.access(path.join(outputDir, "crabline-fake-provider-server.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); await expect( transport.sendInbound({ conversation: { id: "-1001234567890", kind: "group" }, @@ -106,15 +107,13 @@ describe("crabline transport", () => { text: "driver", }); - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - botToken: string; - endpoints: { apiRoot: string }; - }; - const response = await fetch( - `${manifest.endpoints.apiRoot}/bot${manifest.botToken}/getUpdates`, - ); + const config = transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" }); + const telegram = config.channels?.telegram as + | { apiRoot?: string; botToken?: string } + | undefined; + const apiRoot = requireString(telegram?.apiRoot, "Telegram API root"); + const botToken = requireString(telegram?.botToken, "Telegram bot token"); + const response = await fetch(`${apiRoot}/bot${botToken}/getUpdates`); const payload = (await response.json()) as { result?: Array<{ message?: { from?: { id?: number }; text?: string } }>; }; @@ -155,15 +154,13 @@ describe("crabline transport", () => { senderName: "Alice", }); - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - botToken: string; - endpoints: { apiRoot: string }; - }; - const response = await fetch( - `${manifest.endpoints.apiRoot}/bot${manifest.botToken}/getUpdates`, - ); + const config = transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" }); + const telegram = config.channels?.telegram as + | { apiRoot?: string; botToken?: string } + | undefined; + const apiRoot = requireString(telegram?.apiRoot, "Telegram API root"); + const botToken = requireString(telegram?.botToken, "Telegram bot token"); + const response = await fetch(`${apiRoot}/bot${botToken}/getUpdates`); await expect(response.json()).resolves.toMatchObject({ result: [ { @@ -189,21 +186,18 @@ describe("crabline transport", () => { }); try { - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - botToken: string; - endpoints: { apiRoot: string }; - }; + const config = transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" }); + const telegram = config.channels?.telegram as + | { apiRoot?: string; botToken?: string } + | undefined; + const apiRoot = requireString(telegram?.apiRoot, "Telegram API root"); + const botToken = requireString(telegram?.botToken, "Telegram bot token"); const postTelegram = async (method: string, body: Record) => { - const response = await fetch( - `${manifest.endpoints.apiRoot}/bot${manifest.botToken}/${method}`, - { - body: JSON.stringify(body), - headers: { "content-type": "application/json" }, - method: "POST", - }, - ); + const response = await fetch(`${apiRoot}/bot${botToken}/${method}`, { + body: JSON.stringify(body), + headers: { "content-type": "application/json" }, + method: "POST", + }); expect(response.ok).toBe(true); return (await response.json()) as { result: { message_id: number } }; }; @@ -269,13 +263,6 @@ describe("crabline transport", () => { SLACK_BOT_TOKEN: "xoxb-crabline-slack-token", SLACK_SIGNING_SECRET: "crabline-slack-signing-secret", }); - - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - provider?: string; - }; - expect(manifest.provider).toBe("slack"); } finally { await transport.cleanup?.(); } @@ -395,13 +382,6 @@ describe("crabline transport", () => { }); expect(env.CRABLINE_WHATSAPP_ACCESS_TOKEN).toBeUndefined(); expect(env.CRABLINE_WHATSAPP_API_ROOT).toBeUndefined(); - - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - provider?: string; - }; - expect(manifest.provider).toBe("whatsapp"); } finally { await transport.cleanup?.(); } @@ -499,13 +479,11 @@ describe("crabline transport", () => { try { const delivery = transport.buildAgentDelivery({ target: "dm:alice" }); - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - endpoints: { rpcUrl: string }; - }; + const config = transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" }); + const signal = config.channels?.signal as { httpUrl?: string } | undefined; + const signalBaseUrl = requireString(signal?.httpUrl, "Signal HTTP URL"); const { response, release } = await fetchWithSsrFGuard({ - url: manifest.endpoints.rpcUrl, + url: `${signalBaseUrl}/api/v1/rpc`, init: { body: JSON.stringify({ id: "qa-signal-send", @@ -607,21 +585,18 @@ describe("crabline transport", () => { text: "Mattermost baseline marker check.", }); const delivery = transport.buildAgentDelivery({ target: "group:qa-channel" }); - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - botToken: string; - endpoints: { apiRoot: string }; - }; + const env = transport.createRuntimeEnvPatch?.() ?? {}; + const mattermostUrl = requireString(env.MATTERMOST_URL, "Mattermost URL"); + const botToken = requireString(env.MATTERMOST_BOT_TOKEN, "Mattermost bot token"); const { response, release } = await fetchWithSsrFGuard({ - url: `${manifest.endpoints.apiRoot}/posts`, + url: `${mattermostUrl}/api/v4/posts`, init: { body: JSON.stringify({ channel_id: delivery.to.replace(/^channel:/u, ""), message: "assistant via fake mattermost", }), headers: { - authorization: `Bearer ${manifest.botToken}`, + authorization: `Bearer ${botToken}`, "content-type": "application/json", }, method: "POST", @@ -721,18 +696,15 @@ describe("crabline transport", () => { }); const delivery = transport.buildAgentDelivery({ target: `group:${roomId}` }); const providerRoomId = delivery.to.replace(/^room:/u, ""); - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - accessToken: string; - endpoints: { clientApiRoot: string }; - }; + const env = transport.createRuntimeEnvPatch?.() ?? {}; + const matrixBaseUrl = requireString(env.MATRIX_BASE_URL, "Matrix base URL"); + const accessToken = requireString(env.MATRIX_ACCESS_TOKEN, "Matrix access token"); const { response, release } = await fetchWithSsrFGuard({ - url: `${manifest.endpoints.clientApiRoot}/rooms/${encodeURIComponent(providerRoomId)}/send/m.room.message/qa-matrix-send`, + url: `${matrixBaseUrl}/_matrix/client/v3/rooms/${encodeURIComponent(providerRoomId)}/send/m.room.message/qa-matrix-send`, init: { body: JSON.stringify({ body: "assistant via fake matrix", msgtype: "m.text" }), headers: { - authorization: `Bearer ${manifest.accessToken}`, + authorization: `Bearer ${accessToken}`, "content-type": "application/json", }, method: "PUT", @@ -800,14 +772,11 @@ describe("crabline transport", () => { to: "qa-group", }); - const manifest = JSON.parse( - await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"), - ) as { - botToken: string; - endpoints: { apiRoot: string }; - }; + const env = transport.createRuntimeEnvPatch?.() ?? {}; + const zaloApiUrl = requireString(env.ZALO_API_URL, "Zalo API URL"); + const botToken = requireString(env.ZALO_BOT_TOKEN, "Zalo bot token"); const { response, release } = await fetchWithSsrFGuard({ - url: `${manifest.endpoints.apiRoot}/bot${manifest.botToken}/sendMessage`, + url: `${zaloApiUrl}/bot${botToken}/sendMessage`, init: { body: JSON.stringify({ chat_id: delivery.to, diff --git a/extensions/qa-lab/src/crabline-transport.ts b/extensions/qa-lab/src/crabline-transport.ts index cdb10c78cce9..f6a8c62909fe 100644 --- a/extensions/qa-lab/src/crabline-transport.ts +++ b/extensions/qa-lab/src/crabline-transport.ts @@ -3,7 +3,6 @@ import fs from "node:fs/promises"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { - OPENCLAW_CRABLINE_MANIFEST_PATH, startOpenClawCrablineAdapter, type OpenClawCrablineChannelDriverSelection, type OpenClawCrablineInbound, @@ -473,11 +472,6 @@ export async function createQaCrablineTransportAdapter(params: { openclawConfig: {}, recorderPath, }); - await fs.writeFile( - path.join(params.outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), - `${JSON.stringify(adapter.manifest, null, 2)}\n`, - "utf8", - ); const state = createCrablineState({ adapter, diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index 4ed41e11136f..c4a623dfdf5b 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -501,6 +501,99 @@ describe("qa suite", () => { } }); + it("uses Crabline generation artifact paths without rewriting them", async () => { + const outputDir = await tempDirs.makeTempDir("qa-suite-crabline-generation-"); + const capabilityMatrixPath = path.join( + outputDir, + "crabline-generations", + "generation-1", + "capabilities.json", + ); + const smokeArtifactPath = path.join( + outputDir, + "crabline-generations", + "generation-1", + "smoke.json", + ); + await fs.mkdir(path.dirname(capabilityMatrixPath), { recursive: true }); + await fs.writeFile(capabilityMatrixPath, "authoritative capabilities\n", "utf8"); + await fs.writeFile(smokeArtifactPath, "authoritative smoke\n", "utf8"); + + const artifacts = await qaSuiteProgressTesting.writeQaSuiteArtifacts({ + outputDir, + startedAt: new Date("2026-07-12T00:00:00.000Z"), + finishedAt: new Date("2026-07-12T00:01:00.000Z"), + scenarios: [{ name: "Telegram DM", status: "pass", steps: [] }], + scenarioDefinitions: [ + { + ...makeQaSuiteTestScenario("telegram-dm", { + surface: "channel", + }), + coverage: { + primary: ["channels.dm"], + }, + }, + ], + transport: { + id: "qa-channel", + createReportNotes: () => [], + } as unknown as QaTransportAdapter, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + alternateModel: "mock-openai/gpt-5.6-luna-alt", + fastMode: true, + concurrency: 1, + channelDriverSelection: { + capabilityMatrixPath: "crabline-fake-provider-capabilities.json", + channel: "telegram", + channelDriver: "crabline", + smokeArtifactPath: "crabline-fake-provider-smoke.json", + }, + runCrablineChannelDriverSmoke: vi.fn(async () => ({ + capabilityMatrixPath, + capabilityReport: {}, + manifestPath: path.join(outputDir, "crabline-generations", "generation-1", "manifest.json"), + smoke: {}, + smokeArtifactPath, + })), + }); + + await expect(fs.readFile(capabilityMatrixPath, "utf8")).resolves.toBe( + "authoritative capabilities\n", + ); + await expect(fs.readFile(smokeArtifactPath, "utf8")).resolves.toBe("authoritative smoke\n"); + await expect( + fs.access(path.join(outputDir, "crabline-fake-provider-capabilities.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + fs.access(path.join(outputDir, "crabline-fake-provider-smoke.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + + const evidence = JSON.parse(await fs.readFile(artifacts.evidencePath, "utf8")) as { + entries?: Array<{ execution?: { artifacts?: Array<{ kind?: string; path?: string }> } }>; + }; + expect(evidence.entries?.[0]?.execution?.artifacts).toEqual( + expect.arrayContaining([ + { kind: "channel-capability-matrix", path: capabilityMatrixPath, source: "qa-suite" }, + { kind: "channel-driver-smoke", path: smokeArtifactPath, source: "qa-suite" }, + ]), + ); + const summary = JSON.parse(await fs.readFile(artifacts.summaryPath, "utf8")) as { + run?: { + channelCapabilityMatrixPath?: string; + channelDriverSmokePath?: string; + }; + }; + expect(summary.run).toMatchObject({ + channelCapabilityMatrixPath: capabilityMatrixPath, + channelDriverSmokePath: smokeArtifactPath, + }); + expect(artifacts.report).toContain(`Channel capability report: ${capabilityMatrixPath}.`); + expect(artifacts.report).toContain(`Channel driver smoke: ${smokeArtifactPath}.`); + expect(artifacts.report).not.toContain("crabline-fake-provider-capabilities.json"); + expect(artifacts.report).not.toContain("crabline-fake-provider-smoke.json"); + }); + it("arms gateway heap checkpoint env only when requested", () => { expect( qaSuiteProgressTesting.buildQaGatewayHeapCheckpointRuntimeEnvPatch({ diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 387a2758fc7b..0d741062833c 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -97,6 +97,24 @@ type QaSuiteStep = { run: () => Promise; }; +type QaCrablineChannelDriverSmokeResult = Awaited< + ReturnType +> & { + capabilityMatrixPath?: unknown; + smokeArtifactPath?: unknown; +}; + +type QaCrablineChannelDriverArtifactPaths = { + capabilityMatrixPath: string; + smokeArtifactPath: string; +}; + +type QaSuiteChannelDriverSelection = Omit< + OpenClawCrablineChannelDriverSelection, + "capabilityMatrixPath" | "smokeArtifactPath" +> & + QaCrablineChannelDriverArtifactPaths; + function resolveQaSuiteControlUiEnabled(params: { explicit?: boolean; scenarios: ReturnType["scenarios"]; @@ -548,7 +566,7 @@ function buildRuntimeParityScenarioResult(params: { function createQaSuiteReportNotes(params: { transport: QaTransportAdapter; - channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; + channelDriverSelection?: QaSuiteChannelDriverSelection | null; providerMode: QaProviderMode; primaryModel: string; alternateModel: string; @@ -558,7 +576,10 @@ function createQaSuiteReportNotes(params: { }) { return [ ...params.transport.createReportNotes(params), - ...createOpenClawCrablineChannelReportNotes(params.channelDriverSelection), + // Crabline 0.1.9 narrows these paths to legacy filename literals. + ...createOpenClawCrablineChannelReportNotes( + params.channelDriverSelection as OpenClawCrablineChannelDriverSelection | null | undefined, + ), ]; } @@ -695,7 +716,7 @@ export type QaSuiteSummaryJsonParams = { fastMode: boolean; concurrency: number; channelDriver?: QaScorecardChannelDriver | null; - channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; + channelDriverSelection?: QaSuiteChannelDriverSelection | null; scenarioIds?: readonly string[]; runtimePair?: [RuntimeId, RuntimeId]; }; @@ -1041,29 +1062,48 @@ async function writeQaSuiteArtifacts(params: { scenarioIds?: readonly string[]; runtimePair?: [RuntimeId, RuntimeId]; writeEvidenceFile?: boolean; + runCrablineChannelDriverSmoke?: ( + params: Parameters[0], + ) => Promise; }) { const reportPath = path.join(params.outputDir, "qa-suite-report.md"); const summaryPath = path.join(params.outputDir, "qa-suite-summary.json"); const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME); const crablineChannelDriverSelection = params.channelDriverSelection; - const crablineChannelDriverSmoke = crablineChannelDriverSelection - ? await runOpenClawCrablineChannelDriverSmoke({ - outputDir: params.outputDir, - selection: crablineChannelDriverSelection, - }) - : undefined; - const crablineChannelDriverArtifactPaths = crablineChannelDriverSelection - ? [ - { - kind: "channel-capability-matrix", - path: crablineChannelDriverSelection.capabilityMatrixPath, - }, - { - kind: "channel-driver-smoke", - path: crablineChannelDriverSelection.smokeArtifactPath, - }, - ] - : []; + const crablineChannelDriverSmoke: QaCrablineChannelDriverSmokeResult | undefined = + crablineChannelDriverSelection + ? await (params.runCrablineChannelDriverSmoke ?? runOpenClawCrablineChannelDriverSmoke)({ + outputDir: params.outputDir, + selection: crablineChannelDriverSelection, + }) + : undefined; + const authoritativeCapabilityMatrixPath = + typeof crablineChannelDriverSmoke?.capabilityMatrixPath === "string" && + crablineChannelDriverSmoke.capabilityMatrixPath.trim().length > 0 + ? crablineChannelDriverSmoke.capabilityMatrixPath.trim() + : undefined; + const authoritativeSmokeArtifactPath = + typeof crablineChannelDriverSmoke?.smokeArtifactPath === "string" && + crablineChannelDriverSmoke.smokeArtifactPath.trim().length > 0 + ? crablineChannelDriverSmoke.smokeArtifactPath.trim() + : undefined; + const crablineChannelDriverArtifactPaths: QaCrablineChannelDriverArtifactPaths | undefined = + crablineChannelDriverSelection + ? { + capabilityMatrixPath: + authoritativeCapabilityMatrixPath ?? + crablineChannelDriverSelection.capabilityMatrixPath, + smokeArtifactPath: + authoritativeSmokeArtifactPath ?? crablineChannelDriverSelection.smokeArtifactPath, + } + : undefined; + const effectiveChannelDriverSelection: QaSuiteChannelDriverSelection | null | undefined = + crablineChannelDriverSelection && crablineChannelDriverArtifactPaths + ? { + ...crablineChannelDriverSelection, + ...crablineChannelDriverArtifactPaths, + } + : crablineChannelDriverSelection; const report = renderQaMarkdownReport({ title: "OpenClaw QA Scenario Suite", startedAt: params.startedAt, @@ -1075,7 +1115,10 @@ async function writeQaSuiteArtifacts(params: { details: scenario.details, steps: scenario.steps, })) satisfies QaReportScenario[], - notes: createQaSuiteReportNotes(params), + notes: createQaSuiteReportNotes({ + ...params, + channelDriverSelection: effectiveChannelDriverSelection, + }), }); const evidence = params.scenarioDefinitions && params.scenarioDefinitions.length > 0 @@ -1083,7 +1126,18 @@ async function writeQaSuiteArtifacts(params: { artifactPaths: [ { kind: "summary", path: path.basename(summaryPath) }, { kind: "report", path: path.basename(reportPath) }, - ...crablineChannelDriverArtifactPaths, + ...(crablineChannelDriverArtifactPaths + ? [ + { + kind: "channel-capability-matrix", + path: crablineChannelDriverArtifactPaths.capabilityMatrixPath, + }, + { + kind: "channel-driver-smoke", + path: crablineChannelDriverArtifactPaths.smokeArtifactPath, + }, + ] + : []), ], evidenceMode: params.evidenceMode, channelId: params.channelDriverSelection?.channel ?? params.transport.id, @@ -1097,7 +1151,11 @@ async function writeQaSuiteArtifacts(params: { scenarioResults: params.scenarios, }) : undefined; - if (crablineChannelDriverSelection && crablineChannelDriverSmoke) { + if ( + crablineChannelDriverSelection && + crablineChannelDriverSmoke && + !authoritativeCapabilityMatrixPath + ) { await fs.writeFile( path.join(params.outputDir, crablineChannelDriverSelection.capabilityMatrixPath), `${JSON.stringify( @@ -1114,6 +1172,12 @@ async function writeQaSuiteArtifacts(params: { )}\n`, "utf8", ); + } + if ( + crablineChannelDriverSelection && + crablineChannelDriverSmoke && + !authoritativeSmokeArtifactPath + ) { await fs.writeFile( path.join(params.outputDir, crablineChannelDriverSelection.smokeArtifactPath), `${JSON.stringify( @@ -1138,7 +1202,14 @@ async function writeQaSuiteArtifacts(params: { } await fs.writeFile( summaryPath, - `${JSON.stringify(buildQaSuiteSummaryJson(params), null, 2)}\n`, + `${JSON.stringify( + buildQaSuiteSummaryJson({ + ...params, + channelDriverSelection: effectiveChannelDriverSelection, + }), + null, + 2, + )}\n`, "utf8", ); await assertQaSuiteArtifactWritten("report", reportPath);