diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index 083a231cbf7..97228a77b2e 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -1,5 +1,8 @@ +import fs from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; -import { __testing } from "./slack-live.runtime.js"; +import { __testing, runSlackQaLive } from "./slack-live.runtime.js"; describe("Slack live QA runtime helpers", () => { it("resolves env credential payloads", () => { @@ -91,4 +94,34 @@ describe("Slack live QA runtime helpers", () => { }, ]); }); + + it("writes artifacts when Convex credential acquisition fails", async () => { + const outputDir = await fs.mkdtemp(path.join(tmpdir(), "openclaw-slack-qa-")); + const result = await runSlackQaLive({ + credentialRole: "ci", + credentialSource: "convex", + outputDir, + }); + + expect(result.scenarios).toMatchObject([ + { + id: "slack-canary", + status: "fail", + }, + ]); + expect(result.scenarios[0]?.details).toContain("Missing OPENCLAW_QA_CONVEX_SITE_URL"); + await expect(fs.stat(result.reportPath)).resolves.toMatchObject({ + isFile: expect.any(Function), + }); + const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { + channelId: string; + credentials: { kind: string; role?: string; source: string }; + }; + expect(summary.channelId).toBe(""); + expect(summary.credentials).toEqual({ + kind: "slack", + role: "ci", + source: "convex", + }); + }); }); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts index b76aa4c776d..be13f65b0e1 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts @@ -122,6 +122,9 @@ type SlackQaSummary = { startedAt: string; }; +type SlackCredentialLease = Awaited>>; +type SlackCredentialHeartbeat = ReturnType; + const SLACK_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_SLACK_CAPTURE_CONTENT"; const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; const SLACK_QA_ENV_KEYS = [ @@ -212,6 +215,23 @@ function isTruthyOptIn(value: string | undefined) { return normalized === "1" || normalized === "true" || normalized === "yes"; } +function inferSlackCredentialSource( + value: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): "convex" | "env" { + const normalized = + value?.trim().toLowerCase() || env.OPENCLAW_QA_CREDENTIAL_SOURCE?.trim().toLowerCase(); + return normalized === "convex" ? "convex" : "env"; +} + +function inferSlackCredentialRole(value: string | undefined): QaCredentialRole | undefined { + const normalized = value?.trim().toLowerCase(); + if (normalized === "ci" || normalized === "maintainer") { + return normalized; + } + return undefined; +} + function normalizeSlackId(value: string, label: string) { const normalized = value.trim(); if (!/^[A-Z][A-Z0-9]+$/.test(normalized)) { @@ -607,20 +627,8 @@ export async function runSlackQaLive(params: { const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); const sutAccountId = params.sutAccountId?.trim() || "sut"; const scenarios = findScenario(params.scenarioIds); - - const credentialLease = await acquireQaCredentialLease({ - kind: "slack", - source: params.credentialSource, - role: params.credentialRole, - resolveEnvPayload: () => resolveSlackQaRuntimeEnv(), - parsePayload: parseSlackQaCredentialPayload, - }); - const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease); - const assertLeaseHealthy = () => { - leaseHeartbeat.throwIfFailed(); - }; - - const runtimeEnv = credentialLease.payload; + const requestedCredentialSource = inferSlackCredentialSource(params.credentialSource); + const requestedCredentialRole = inferSlackCredentialRole(params.credentialRole); const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]); const includeObservedMessageContent = isTruthyOptIn(process.env[SLACK_QA_CAPTURE_CONTENT_ENV]); const startedAt = new Date().toISOString(); @@ -629,18 +637,37 @@ export async function runSlackQaLive(params: { const cleanupIssues: string[] = []; const gatewayDebugDirPath = path.join(outputDir, "gateway-debug"); let preservedGatewayDebugArtifacts = false; + let credentialLease: SlackCredentialLease | undefined; + let leaseHeartbeat: SlackCredentialHeartbeat | undefined; + let runtimeEnv: SlackQaRuntimeEnv | undefined; try { + credentialLease = await acquireQaCredentialLease({ + kind: "slack", + source: params.credentialSource, + role: params.credentialRole, + resolveEnvPayload: () => resolveSlackQaRuntimeEnv(), + parsePayload: parseSlackQaCredentialPayload, + }); + leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease); + const assertLeaseHealthy = () => { + leaseHeartbeat?.throwIfFailed(); + }; + const activeRuntimeEnv = credentialLease.payload; + runtimeEnv = activeRuntimeEnv; + const [driverIdentity, sutIdentity] = await Promise.all([ - getSlackIdentity(runtimeEnv.driverBotToken), - getSlackIdentity(runtimeEnv.sutBotToken), + getSlackIdentity(activeRuntimeEnv.driverBotToken), + getSlackIdentity(activeRuntimeEnv.sutBotToken), ]); if (driverIdentity.userId === sutIdentity.userId) { throw new Error("Slack QA requires two distinct bots for driver and SUT."); } - const driverClient = createSlackWriteClient(runtimeEnv.driverBotToken, { timeout: 15_000 }); - const sutReadClient = createSlackWebClient(runtimeEnv.sutBotToken, { timeout: 15_000 }); + const driverClient = createSlackWriteClient(activeRuntimeEnv.driverBotToken, { + timeout: 15_000, + }); + const sutReadClient = createSlackWebClient(activeRuntimeEnv.sutBotToken, { timeout: 15_000 }); const gatewayHarness = await startQaLiveLaneGateway({ repoRoot, transport: { @@ -655,11 +682,11 @@ export async function runSlackQaLive(params: { controlUiEnabled: false, mutateConfig: (cfg) => buildSlackQaConfig(cfg, { - channelId: runtimeEnv.channelId, + channelId: activeRuntimeEnv.channelId, driverBotUserId: driverIdentity.userId, sutAccountId, - sutAppToken: runtimeEnv.sutAppToken, - sutBotToken: runtimeEnv.sutBotToken, + sutAppToken: activeRuntimeEnv.sutAppToken, + sutBotToken: activeRuntimeEnv.sutBotToken, }), }); try { @@ -671,13 +698,13 @@ export async function runSlackQaLive(params: { const requestStartedAt = new Date(); try { const sent = await sendSlackChannelMessage({ - channelId: runtimeEnv.channelId, + channelId: activeRuntimeEnv.channelId, client: driverClient, text: scenarioRun.input, }); if (scenarioRun.expectReply) { const reply = await waitForSlackScenarioReply({ - channelId: runtimeEnv.channelId, + channelId: activeRuntimeEnv.channelId, client: sutReadClient, matchText: scenarioRun.matchText, observedMessages, @@ -700,7 +727,7 @@ export async function runSlackQaLive(params: { }); } else { await waitForSlackNoReply({ - channelId: runtimeEnv.channelId, + channelId: activeRuntimeEnv.channelId, client: sutReadClient, matchText: scenarioRun.matchText, observedMessages, @@ -760,15 +787,19 @@ export async function runSlackQaLive(params: { details: formatErrorMessage(error), }); } finally { - try { - await leaseHeartbeat.stop(); - } catch (error) { - appendLiveLaneIssue(cleanupIssues, "credential heartbeat stop failed", error); + if (leaseHeartbeat) { + try { + await leaseHeartbeat.stop(); + } catch (error) { + appendLiveLaneIssue(cleanupIssues, "credential heartbeat stop failed", error); + } } - try { - await credentialLease.release(); - } catch (error) { - appendLiveLaneIssue(cleanupIssues, "credential release failed", error); + if (credentialLease) { + try { + await credentialLease.release(); + } catch (error) { + appendLiveLaneIssue(cleanupIssues, "credential release failed", error); + } } } @@ -779,14 +810,24 @@ export async function runSlackQaLive(params: { const passed = scenarioResults.filter((entry) => entry.status === "pass").length; const failed = scenarioResults.filter((entry) => entry.status === "fail").length; const summary: SlackQaSummary = { - credentials: { - source: credentialLease.source, - kind: credentialLease.kind, - role: credentialLease.role, - credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId, - ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId, - }, - channelId: redactPublicMetadata ? "" : runtimeEnv.channelId, + credentials: credentialLease + ? { + source: credentialLease.source, + kind: credentialLease.kind, + role: credentialLease.role, + credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId, + ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId, + } + : { + source: requestedCredentialSource, + kind: "slack", + role: requestedCredentialRole, + }, + channelId: runtimeEnv + ? redactPublicMetadata + ? "" + : runtimeEnv.channelId + : "", startedAt, finishedAt, cleanupIssues, @@ -813,9 +854,9 @@ export async function runSlackQaLive(params: { await fs.writeFile( reportPath, `${renderSlackQaMarkdown({ - channelId: runtimeEnv.channelId, + channelId: runtimeEnv?.channelId ?? "", cleanupIssues, - credentialSource: credentialLease.source, + credentialSource: credentialLease?.source ?? requestedCredentialSource, finishedAt, gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined, redactMetadata: redactPublicMetadata,