mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:50:43 +00:00
feat(qa-lab): add control ui qa-channel roundtrip scenario
This commit is contained in:
Peter Steinberger
@@ -131,6 +131,11 @@ the source of truth for one test run and should define:
|
||||
- optional gateway config patch
|
||||
- the executable `qa-flow`
|
||||
|
||||
The reusable runtime surface that backs `qa-flow` is allowed to stay generic
|
||||
and cross-cutting. For example, markdown scenarios can combine transport-side
|
||||
helpers with browser-side helpers that drive the embedded Control UI through the
|
||||
Gateway `browser.request` seam without adding a special-case runner.
|
||||
|
||||
The baseline list should stay broad enough to cover:
|
||||
|
||||
- DM and channel chat
|
||||
|
||||
@@ -115,6 +115,25 @@ describe("buildQaRuntimeEnv", () => {
|
||||
expect(env.OPENCLAW_STATE_DIR).toBe("/tmp/openclaw-qa/state");
|
||||
});
|
||||
|
||||
it("can forward host HOME for browser-backed QA runs while keeping OpenClaw home sandboxed", async () => {
|
||||
const hostHome = await mkdtemp(path.join(os.tmpdir(), "qa-host-home-"));
|
||||
cleanups.push(async () => {
|
||||
await rm(hostHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const env = buildQaRuntimeEnv({
|
||||
...createParams({
|
||||
HOME: hostHome,
|
||||
}),
|
||||
providerMode: "mock-openai",
|
||||
forwardHostHome: true,
|
||||
});
|
||||
|
||||
expect(env.HOME).toBe(hostHome);
|
||||
expect(env.OPENCLAW_HOME).toBe("/tmp/openclaw-qa/home");
|
||||
expect(env.OPENCLAW_STATE_DIR).toBe("/tmp/openclaw-qa/state");
|
||||
});
|
||||
|
||||
it("preserves the live Anthropic key for live Claude CLI runs without writing it into config", async () => {
|
||||
const hostHome = await mkdtemp(path.join(os.tmpdir(), "qa-host-home-"));
|
||||
cleanups.push(async () => {
|
||||
|
||||
@@ -295,6 +295,7 @@ export function buildQaRuntimeEnv(params: {
|
||||
configPath: string;
|
||||
gatewayToken: string;
|
||||
homeDir: string;
|
||||
forwardHostHome?: boolean;
|
||||
stateDir: string;
|
||||
xdgConfigHome: string;
|
||||
xdgDataHome: string;
|
||||
@@ -307,9 +308,12 @@ export function buildQaRuntimeEnv(params: {
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
}) {
|
||||
const baseEnv = params.baseEnv ?? process.env;
|
||||
const forwardedHostHome = params.forwardHostHome
|
||||
? baseEnv.HOME?.trim() || os.homedir()
|
||||
: undefined;
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...baseEnv,
|
||||
HOME: params.homeDir,
|
||||
HOME: forwardedHostHome ?? params.homeDir,
|
||||
...(params.providerMode === "live-frontier"
|
||||
? resolveQaLiveCliAuthEnv(baseEnv, {
|
||||
forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli,
|
||||
@@ -837,6 +841,7 @@ export async function startQaGatewayChild(params: {
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
controlUiEnabled?: boolean;
|
||||
enabledPluginIds?: string[];
|
||||
forwardHostHome?: boolean;
|
||||
mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
|
||||
}) {
|
||||
const tempRoot = await fs.mkdtemp(
|
||||
@@ -969,6 +974,7 @@ export async function startQaGatewayChild(params: {
|
||||
configPath,
|
||||
gatewayToken,
|
||||
homeDir,
|
||||
forwardHostHome: params.forwardHostHome,
|
||||
stateDir,
|
||||
xdgConfigHome,
|
||||
xdgDataHome,
|
||||
|
||||
@@ -81,6 +81,12 @@ describe("qa scenario catalog", () => {
|
||||
expect(fanoutConfig?.expectedReplyGroups?.flat()).toContain("subagent-2: ok");
|
||||
});
|
||||
|
||||
it("loads scenario-declared gateway runtime options from markdown", () => {
|
||||
const scenario = readQaScenarioById("control-ui-qa-channel-image-roundtrip");
|
||||
|
||||
expect(scenario.gatewayRuntime?.forwardHostHome).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the character eval scenario natural and task-shaped", () => {
|
||||
const characterConfig = readQaScenarioExecutionConfig("character-vibes-gollum") as
|
||||
| {
|
||||
|
||||
@@ -51,6 +51,10 @@ const qaScenarioExecutionSchema = z.object({
|
||||
config: qaScenarioConfigSchema.optional(),
|
||||
});
|
||||
|
||||
const qaScenarioGatewayRuntimeSchema = z.object({
|
||||
forwardHostHome: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const qaFlowCallActionSchema = z.object({
|
||||
call: z.string().trim().min(1),
|
||||
args: z.array(z.unknown()).optional(),
|
||||
@@ -137,6 +141,7 @@ const qaSeedScenarioSchema = z.object({
|
||||
successCriteria: z.array(z.string().trim().min(1)).min(1),
|
||||
plugins: z.array(z.string().trim().min(1)).optional(),
|
||||
gatewayConfigPatch: z.record(z.string(), z.unknown()).optional(),
|
||||
gatewayRuntime: qaScenarioGatewayRuntimeSchema.optional(),
|
||||
docsRefs: z.array(z.string().trim().min(1)).optional(),
|
||||
codeRefs: z.array(z.string().trim().min(1)).optional(),
|
||||
execution: qaScenarioExecutionSchema.optional(),
|
||||
|
||||
@@ -30,6 +30,16 @@ function createDeps(overrides?: Partial<QaScenarioRuntimeDeps>): QaScenarioRunti
|
||||
waitForGatewayHealthy: fn,
|
||||
waitForTransportReady: fn,
|
||||
waitForQaChannelReady: fn,
|
||||
browserRequest: fn,
|
||||
waitForBrowserReady: fn,
|
||||
browserOpenTab: fn,
|
||||
browserSnapshot: fn,
|
||||
browserAct: fn,
|
||||
webOpenPage: fn,
|
||||
webWait: fn,
|
||||
webType: fn,
|
||||
webSnapshot: fn,
|
||||
webEvaluate: fn,
|
||||
waitForConfigRestartSettle: fn,
|
||||
patchConfig: fn,
|
||||
applyConfig: fn,
|
||||
@@ -130,6 +140,16 @@ describe("createQaScenarioRuntimeApi", () => {
|
||||
expect(api.config).toEqual({ expected: "value" });
|
||||
expect(api.waitForCondition).toBe(waitForCondition);
|
||||
expect(api.waitForChannelReady).toBe(api.waitForTransportReady);
|
||||
expect(api.browserRequest).toBeDefined();
|
||||
expect(api.waitForBrowserReady).toBeDefined();
|
||||
expect(api.browserOpenTab).toBeDefined();
|
||||
expect(api.browserSnapshot).toBeDefined();
|
||||
expect(api.browserAct).toBeDefined();
|
||||
expect(api.webOpenPage).toBeDefined();
|
||||
expect(api.webWait).toBeDefined();
|
||||
expect(api.webType).toBeDefined();
|
||||
expect(api.webSnapshot).toBeDefined();
|
||||
expect(api.webEvaluate).toBeDefined();
|
||||
expect(api.getTransportSnapshot()).toEqual(state.getSnapshot());
|
||||
expect(api.imageUnderstandingPngBase64).toBe("png-small");
|
||||
|
||||
|
||||
@@ -42,6 +42,16 @@ export type QaScenarioRuntimeDeps = {
|
||||
waitForGatewayHealthy: QaScenarioRuntimeFunction;
|
||||
waitForTransportReady: QaScenarioRuntimeFunction;
|
||||
waitForQaChannelReady: QaScenarioRuntimeFunction;
|
||||
browserRequest: QaScenarioRuntimeFunction;
|
||||
waitForBrowserReady: QaScenarioRuntimeFunction;
|
||||
browserOpenTab: QaScenarioRuntimeFunction;
|
||||
browserSnapshot: QaScenarioRuntimeFunction;
|
||||
browserAct: QaScenarioRuntimeFunction;
|
||||
webOpenPage: QaScenarioRuntimeFunction;
|
||||
webWait: QaScenarioRuntimeFunction;
|
||||
webType: QaScenarioRuntimeFunction;
|
||||
webSnapshot: QaScenarioRuntimeFunction;
|
||||
webEvaluate: QaScenarioRuntimeFunction;
|
||||
waitForConfigRestartSettle: QaScenarioRuntimeFunction;
|
||||
patchConfig: QaScenarioRuntimeFunction;
|
||||
applyConfig: QaScenarioRuntimeFunction;
|
||||
@@ -116,6 +126,16 @@ export type QaScenarioRuntimeApi<
|
||||
waitForTransportReady: TDeps["waitForTransportReady"];
|
||||
waitForChannelReady: TDeps["waitForTransportReady"];
|
||||
waitForQaChannelReady: TDeps["waitForQaChannelReady"];
|
||||
browserRequest: TDeps["browserRequest"];
|
||||
waitForBrowserReady: TDeps["waitForBrowserReady"];
|
||||
browserOpenTab: TDeps["browserOpenTab"];
|
||||
browserSnapshot: TDeps["browserSnapshot"];
|
||||
browserAct: TDeps["browserAct"];
|
||||
webOpenPage: TDeps["webOpenPage"];
|
||||
webWait: TDeps["webWait"];
|
||||
webType: TDeps["webType"];
|
||||
webSnapshot: TDeps["webSnapshot"];
|
||||
webEvaluate: TDeps["webEvaluate"];
|
||||
waitForConfigRestartSettle: TDeps["waitForConfigRestartSettle"];
|
||||
patchConfig: TDeps["patchConfig"];
|
||||
applyConfig: TDeps["applyConfig"];
|
||||
@@ -205,6 +225,16 @@ export function createQaScenarioRuntimeApi<
|
||||
waitForTransportReady: params.deps.waitForTransportReady,
|
||||
waitForChannelReady: params.deps.waitForTransportReady,
|
||||
waitForQaChannelReady: params.deps.waitForQaChannelReady,
|
||||
browserRequest: params.deps.browserRequest,
|
||||
waitForBrowserReady: params.deps.waitForBrowserReady,
|
||||
browserOpenTab: params.deps.browserOpenTab,
|
||||
browserSnapshot: params.deps.browserSnapshot,
|
||||
browserAct: params.deps.browserAct,
|
||||
webOpenPage: params.deps.webOpenPage,
|
||||
webWait: params.deps.webWait,
|
||||
webType: params.deps.webType,
|
||||
webSnapshot: params.deps.webSnapshot,
|
||||
webEvaluate: params.deps.webEvaluate,
|
||||
waitForConfigRestartSettle: params.deps.waitForConfigRestartSettle,
|
||||
patchConfig: params.deps.patchConfig,
|
||||
applyConfig: params.deps.applyConfig,
|
||||
|
||||
@@ -11,6 +11,7 @@ describe("qa suite failure reply handling", () => {
|
||||
config?: Record<string, unknown>,
|
||||
plugins?: string[],
|
||||
gatewayConfigPatch?: Record<string, unknown>,
|
||||
gatewayRuntime?: { forwardHostHome?: boolean },
|
||||
): Parameters<typeof qaSuiteTesting.selectQaSuiteScenarios>[0]["scenarios"][number] =>
|
||||
({
|
||||
id,
|
||||
@@ -20,6 +21,7 @@ describe("qa suite failure reply handling", () => {
|
||||
successCriteria: ["test"],
|
||||
plugins,
|
||||
gatewayConfigPatch,
|
||||
gatewayRuntime,
|
||||
sourcePath: `qa/scenarios/${id}.md`,
|
||||
execution: {
|
||||
kind: "flow",
|
||||
@@ -199,6 +201,19 @@ describe("qa suite failure reply handling", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("collects gateway runtime options across selected scenarios", () => {
|
||||
const scenarios = [
|
||||
makeScenario("plain"),
|
||||
makeScenario("browser-ui", undefined, ["browser"], undefined, {
|
||||
forwardHostHome: true,
|
||||
}),
|
||||
];
|
||||
|
||||
expect(qaSuiteTesting.collectQaSuiteGatewayRuntimeOptions(scenarios)).toEqual({
|
||||
forwardHostHome: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("filters provider-specific scenarios from an implicit live lane", () => {
|
||||
const scenarios = [
|
||||
makeScenario("generic"),
|
||||
|
||||
@@ -15,6 +15,13 @@ import {
|
||||
import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import {
|
||||
callQaBrowserRequest,
|
||||
qaBrowserAct,
|
||||
qaBrowserOpenTab,
|
||||
qaBrowserSnapshot,
|
||||
waitForQaBrowserReady,
|
||||
} from "./browser-runtime.js";
|
||||
import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "./cli-paths.js";
|
||||
import { waitForCronRunCompletion } from "./cron-run-wait.js";
|
||||
import {
|
||||
@@ -60,6 +67,14 @@ import { qaChannelPlugin, type QaBusMessage } from "./runtime-api.js";
|
||||
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
|
||||
import { runScenarioFlow } from "./scenario-flow-runner.js";
|
||||
import { createQaScenarioRuntimeApi } from "./scenario-runtime-api.js";
|
||||
import {
|
||||
closeAllQaWebSessions,
|
||||
qaWebEvaluate,
|
||||
qaWebOpenPage,
|
||||
qaWebSnapshot,
|
||||
qaWebType,
|
||||
qaWebWait,
|
||||
} from "./web-runtime.js";
|
||||
|
||||
type QaSuiteStep = {
|
||||
name: string;
|
||||
@@ -313,6 +328,18 @@ function collectQaSuiteGatewayConfigPatch(
|
||||
return merged;
|
||||
}
|
||||
|
||||
function collectQaSuiteGatewayRuntimeOptions(
|
||||
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"],
|
||||
) {
|
||||
let forwardHostHome = false;
|
||||
for (const scenario of scenarios) {
|
||||
if (scenario.gatewayRuntime?.forwardHostHome === true) {
|
||||
forwardHostHome = true;
|
||||
}
|
||||
}
|
||||
return forwardHostHome ? { forwardHostHome: true } : undefined;
|
||||
}
|
||||
|
||||
function liveTurnTimeoutMs(env: QaSuiteEnvironment, fallbackMs: number) {
|
||||
return resolveQaLiveTurnTimeoutMs(env, fallbackMs);
|
||||
}
|
||||
@@ -1236,6 +1263,16 @@ function createScenarioFlowApi(
|
||||
waitForGatewayHealthy,
|
||||
waitForTransportReady,
|
||||
waitForQaChannelReady,
|
||||
browserRequest: callQaBrowserRequest,
|
||||
waitForBrowserReady: waitForQaBrowserReady,
|
||||
browserOpenTab: qaBrowserOpenTab,
|
||||
browserSnapshot: qaBrowserSnapshot,
|
||||
browserAct: qaBrowserAct,
|
||||
webOpenPage: qaWebOpenPage,
|
||||
webWait: qaWebWait,
|
||||
webType: qaWebType,
|
||||
webSnapshot: qaWebSnapshot,
|
||||
webEvaluate: qaWebEvaluate,
|
||||
waitForConfigRestartSettle,
|
||||
patchConfig,
|
||||
applyConfig,
|
||||
@@ -1284,6 +1321,7 @@ function createScenarioFlowApi(
|
||||
|
||||
export const qaSuiteTesting = {
|
||||
collectQaSuiteGatewayConfigPatch,
|
||||
collectQaSuiteGatewayRuntimeOptions,
|
||||
collectQaSuitePluginIds,
|
||||
createScenarioWaitForCondition,
|
||||
findFailureOutboundMessage,
|
||||
@@ -1397,6 +1435,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
});
|
||||
const enabledPluginIds = collectQaSuitePluginIds(selectedCatalogScenarios);
|
||||
const gatewayConfigPatch = collectQaSuiteGatewayConfigPatch(selectedCatalogScenarios);
|
||||
const gatewayRuntimeOptions = collectQaSuiteGatewayRuntimeOptions(selectedCatalogScenarios);
|
||||
const concurrency = normalizeQaSuiteConcurrency(
|
||||
params?.concurrency,
|
||||
selectedCatalogScenarios.length,
|
||||
@@ -1594,6 +1633,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
claudeCliAuthMode: params?.claudeCliAuthMode,
|
||||
controlUiEnabled: params?.controlUiEnabled ?? true,
|
||||
enabledPluginIds,
|
||||
forwardHostHome: gatewayRuntimeOptions?.forwardHostHome,
|
||||
mutateConfig: gatewayConfigPatch
|
||||
? (cfg) => applyQaMergePatch(cfg, gatewayConfigPatch) as OpenClawConfig
|
||||
: undefined,
|
||||
@@ -1606,9 +1646,9 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
lab,
|
||||
mock,
|
||||
gateway,
|
||||
cfg: transport.createGatewayConfig({
|
||||
baseUrl: lab.listenUrl,
|
||||
}),
|
||||
// Markdown scenarios should see the full staged gateway config, not just
|
||||
// the transport fragment. Routing/session/plugin assertions depend on it.
|
||||
cfg: gateway.cfg,
|
||||
transport,
|
||||
repoRoot,
|
||||
providerMode,
|
||||
@@ -1717,6 +1757,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime");
|
||||
throw error;
|
||||
} finally {
|
||||
await closeAllQaWebSessions();
|
||||
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1" || false;
|
||||
await gateway.stop({
|
||||
keepTemp,
|
||||
|
||||
270
qa/scenarios/control-ui-qa-channel-image-roundtrip.md
Normal file
270
qa/scenarios/control-ui-qa-channel-image-roundtrip.md
Normal file
@@ -0,0 +1,270 @@
|
||||
# Control UI plus qa-channel image roundtrip
|
||||
|
||||
```yaml qa-scenario
|
||||
id: control-ui-qa-channel-image-roundtrip
|
||||
title: Control UI plus qa-channel image roundtrip
|
||||
surface: control-ui
|
||||
objective: Verify the embedded Control UI can observe a qa-channel-backed session while the fake channel injects text and image turns that the agent answers correctly.
|
||||
successCriteria:
|
||||
- Control UI opens directly on the target qa-channel session.
|
||||
- A text prompt delivered through qa-channel produces a correct outbound reply.
|
||||
- A later qa-channel image message produces a correct image-aware reply.
|
||||
- The Control UI transcript shows both transport-side prompts and both final answers.
|
||||
docsRefs:
|
||||
- docs/concepts/qa-e2e-automation.md
|
||||
- docs/channels/qa-channel.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/scenario-runtime-api.ts
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/web-runtime.ts
|
||||
- ui/src/ui/views/chat.ts
|
||||
gatewayRuntime:
|
||||
forwardHostHome: true
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Open the Control UI on a qa-channel session with the generic QA web driver, inject text and image turns through qa-channel, and verify the replies in both the transport log and the UI transcript.
|
||||
config:
|
||||
conversationId: control-ui-e2e
|
||||
textPrompt: "Control UI bridge check. Marker exact marker: `ui bridge armed`"
|
||||
uiExpectedNeedle: ui bridge armed
|
||||
imagePrompt: "Image understanding check: describe the top and bottom colors in the attached image in one short sentence."
|
||||
imagePromptNeedle: image understanding check
|
||||
requiredColorGroups:
|
||||
- [red, scarlet, crimson]
|
||||
- [blue, azure, teal, cyan, aqua]
|
||||
```
|
||||
|
||||
```yaml qa-flow
|
||||
steps:
|
||||
- name: opens control ui on the qa-channel-backed session
|
||||
actions:
|
||||
- call: reset
|
||||
- call: waitForGatewayHealthy
|
||||
args:
|
||||
- ref: env
|
||||
- expr: liveTurnTimeoutMs(env, 60000)
|
||||
- call: waitForQaChannelReady
|
||||
args:
|
||||
- ref: env
|
||||
- expr: liveTurnTimeoutMs(env, 60000)
|
||||
- call: fetchJson
|
||||
saveAs: bootstrap
|
||||
args:
|
||||
- expr: "`${lab.baseUrl}/api/bootstrap`"
|
||||
- assert:
|
||||
expr: "Boolean(bootstrap.controlUiEmbeddedUrl)"
|
||||
message: qa-lab bootstrap did not expose controlUiEmbeddedUrl
|
||||
- set: uiSessionKey
|
||||
value:
|
||||
expr: "buildAgentSessionKey({ agentId: env.cfg.agents?.list?.find((agent) => agent.default)?.id ?? env.cfg.agents?.list?.[0]?.id ?? 'main', channel: 'qa-channel', accountId: 'default', peer: { kind: 'direct', id: config.conversationId }, dmScope: env.cfg.session?.dmScope, identityLinks: env.cfg.session?.identityLinks })"
|
||||
- set: controlUiChatUrl
|
||||
value:
|
||||
expr: "(() => { const url = new URL(String(bootstrap.controlUiEmbeddedUrl)); url.pathname = `${url.pathname.replace(/\\/$/, '')}/chat`; url.searchParams.set('session', uiSessionKey); return url.toString(); })()"
|
||||
- call: webOpenPage
|
||||
saveAs: uiTab
|
||||
args:
|
||||
- url:
|
||||
ref: controlUiChatUrl
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 60000)
|
||||
- set: uiPageId
|
||||
value:
|
||||
expr: "uiTab.pageId"
|
||||
- call: webWait
|
||||
args:
|
||||
- pageId:
|
||||
ref: uiPageId
|
||||
selector: textarea
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 45000)
|
||||
- call: waitForCondition
|
||||
saveAs: uiReadySnapshot
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiPageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); return text.includes('ready to chat') ? snapshot : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 45000)
|
||||
- 500
|
||||
- assert:
|
||||
expr: "Boolean(uiPageId)"
|
||||
message: control ui page was not available
|
||||
detailsExpr: "uiReadySnapshot.text"
|
||||
- name: text injected through qa-channel gets a correct transport reply
|
||||
actions:
|
||||
- set: firstInboundStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'inbound').length"
|
||||
- set: firstOutboundStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- call: injectInboundMessage
|
||||
args:
|
||||
- accountId: default
|
||||
conversation:
|
||||
id:
|
||||
expr: config.conversationId
|
||||
kind: direct
|
||||
senderId:
|
||||
expr: config.conversationId
|
||||
senderName: Control UI QA
|
||||
text:
|
||||
expr: config.textPrompt
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: uiOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === config.conversationId && normalizeLowercaseStringOrEmpty(candidate.text).includes(config.uiExpectedNeedle)"
|
||||
- expr: liveTurnTimeoutMs(env, 45000)
|
||||
- sinceIndex:
|
||||
ref: firstOutboundStartIndex
|
||||
- call: readRawQaSessionStore
|
||||
saveAs: rawSessionStore
|
||||
args:
|
||||
- ref: env
|
||||
- set: rawSessionStoreKeys
|
||||
value:
|
||||
expr: "Object.keys(rawSessionStore)"
|
||||
detailsExpr: "`${uiOutbound.text}\\nSTORE:${JSON.stringify(rawSessionStoreKeys)}`"
|
||||
- name: text injected through qa-channel renders in a fresh control ui load
|
||||
actions:
|
||||
- call: webOpenPage
|
||||
saveAs: uiAckTab
|
||||
args:
|
||||
- url:
|
||||
ref: controlUiChatUrl
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 60000)
|
||||
- set: uiAckPageId
|
||||
value:
|
||||
expr: "uiAckTab.pageId"
|
||||
- call: webWait
|
||||
args:
|
||||
- pageId:
|
||||
ref: uiAckPageId
|
||||
selector: textarea
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 45000)
|
||||
- try:
|
||||
actions:
|
||||
- call: waitForCondition
|
||||
saveAs: uiAckSnapshot
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiAckPageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); return text.includes(config.uiExpectedNeedle) && text.includes('control ui bridge check') ? snapshot : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 45000)
|
||||
- 500
|
||||
catch:
|
||||
- call: webSnapshot
|
||||
saveAs: uiAckFailureSnapshot
|
||||
args:
|
||||
- pageId:
|
||||
ref: uiAckPageId
|
||||
maxChars: 12000
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 15000)
|
||||
- call: webEvaluate
|
||||
saveAs: uiAckFailureState
|
||||
args:
|
||||
- pageId:
|
||||
ref: uiAckPageId
|
||||
expression: "(() => { const app = document.querySelector('openclaw-app'); return app ? { sessionKey: app.sessionKey, settingsSessionKey: app.settings?.sessionKey, lastActiveSessionKey: app.settings?.lastActiveSessionKey, chatMessages: Array.isArray(app.chatMessages) ? app.chatMessages.length : null, chatLoading: app.chatLoading, lastError: app.lastError, connected: app.connected } : null; })()"
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 15000)
|
||||
- throw:
|
||||
expr: "`control ui text transcript missing after fresh load. state=${JSON.stringify(uiAckFailureState)} snapshot: ${uiAckFailureSnapshot.text}`"
|
||||
detailsExpr: "uiAckSnapshot.text"
|
||||
- name: image injected through qa-channel gets a correct transport reply
|
||||
actions:
|
||||
- set: secondOutboundStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- call: injectInboundMessage
|
||||
args:
|
||||
- accountId: default
|
||||
conversation:
|
||||
id:
|
||||
expr: config.conversationId
|
||||
kind: direct
|
||||
senderId:
|
||||
expr: config.conversationId
|
||||
senderName: Control UI QA
|
||||
text:
|
||||
expr: config.imagePrompt
|
||||
attachments:
|
||||
- kind: image
|
||||
mimeType: image/png
|
||||
fileName: red-top-blue-bottom.png
|
||||
altText: red on top blue on bottom
|
||||
contentBase64:
|
||||
expr: imageUnderstandingValidPngBase64
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: imageOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === config.conversationId && config.requiredColorGroups.every((group) => group.some((color) => normalizeLowercaseStringOrEmpty(candidate.text).includes(color)))"
|
||||
- expr: liveTurnTimeoutMs(env, 45000)
|
||||
- sinceIndex:
|
||||
ref: secondOutboundStartIndex
|
||||
- set: missingColorGroup
|
||||
value:
|
||||
expr: "config.requiredColorGroups.find((group) => !group.some((color) => normalizeLowercaseStringOrEmpty(imageOutbound.text).includes(color)))"
|
||||
- assert:
|
||||
expr: "!missingColorGroup"
|
||||
message:
|
||||
expr: "`missing expected colors in image reply: ${imageOutbound.text}`"
|
||||
detailsExpr: "imageOutbound.text"
|
||||
- name: image injected through qa-channel renders in a fresh control ui load
|
||||
actions:
|
||||
- call: webOpenPage
|
||||
saveAs: uiImageTab
|
||||
args:
|
||||
- url:
|
||||
ref: controlUiChatUrl
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 60000)
|
||||
- set: uiImagePageId
|
||||
value:
|
||||
expr: "uiImageTab.pageId"
|
||||
- call: webWait
|
||||
args:
|
||||
- pageId:
|
||||
ref: uiImagePageId
|
||||
selector: textarea
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 45000)
|
||||
- try:
|
||||
actions:
|
||||
- call: waitForCondition
|
||||
saveAs: uiImageSnapshot
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiImagePageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); const hasPrompt = text.includes(config.imagePromptNeedle); const hasColors = config.requiredColorGroups.every((group) => group.some((color) => text.includes(color))); return hasPrompt && hasColors ? snapshot : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 45000)
|
||||
- 500
|
||||
catch:
|
||||
- call: webSnapshot
|
||||
saveAs: uiImageFailureSnapshot
|
||||
args:
|
||||
- pageId:
|
||||
ref: uiImagePageId
|
||||
maxChars: 12000
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 15000)
|
||||
- call: webEvaluate
|
||||
saveAs: uiImageFailureState
|
||||
args:
|
||||
- pageId:
|
||||
ref: uiImagePageId
|
||||
expression: "(() => { const app = document.querySelector('openclaw-app'); return app ? { sessionKey: app.sessionKey, settingsSessionKey: app.settings?.sessionKey, lastActiveSessionKey: app.settings?.lastActiveSessionKey, chatMessages: Array.isArray(app.chatMessages) ? app.chatMessages.length : null, chatLoading: app.chatLoading, lastError: app.lastError, connected: app.connected } : null; })()"
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 15000)
|
||||
- throw:
|
||||
expr: "`control ui image transcript missing after fresh load. state=${JSON.stringify(uiImageFailureState)} snapshot: ${uiImageFailureSnapshot.text}`"
|
||||
detailsExpr: "uiImageSnapshot.text"
|
||||
```
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const loadSessionsMock = vi.fn();
|
||||
const loadChatHistoryMock = vi.fn();
|
||||
|
||||
vi.mock("./app-chat.ts", () => ({
|
||||
CHAT_SESSIONS_ACTIVE_MINUTES: 10,
|
||||
@@ -24,7 +25,7 @@ vi.mock("./controllers/assistant-identity.ts", () => ({
|
||||
loadAssistantIdentity: vi.fn(),
|
||||
}));
|
||||
vi.mock("./controllers/chat.ts", () => ({
|
||||
loadChatHistory: vi.fn(),
|
||||
loadChatHistory: loadChatHistoryMock,
|
||||
handleChatEvent: vi.fn(() => "idle"),
|
||||
}));
|
||||
vi.mock("./controllers/devices.ts", () => ({
|
||||
@@ -124,6 +125,39 @@ describe("handleGatewayEvent sessions.changed", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleGatewayEvent session.message", () => {
|
||||
it("reloads chat history for the active session", () => {
|
||||
loadChatHistoryMock.mockReset();
|
||||
const host = createHost();
|
||||
host.sessionKey = "agent:qa:main";
|
||||
|
||||
handleGatewayEvent(host, {
|
||||
type: "event",
|
||||
event: "session.message",
|
||||
payload: { sessionKey: "agent:qa:main" },
|
||||
seq: 1,
|
||||
});
|
||||
|
||||
expect(loadChatHistoryMock).toHaveBeenCalledTimes(1);
|
||||
expect(loadChatHistoryMock).toHaveBeenCalledWith(host);
|
||||
});
|
||||
|
||||
it("ignores transcript updates for other sessions", () => {
|
||||
loadChatHistoryMock.mockReset();
|
||||
const host = createHost();
|
||||
host.sessionKey = "agent:qa:main";
|
||||
|
||||
handleGatewayEvent(host, {
|
||||
type: "event",
|
||||
event: "session.message",
|
||||
payload: { sessionKey: "agent:qa:other" },
|
||||
seq: 1,
|
||||
});
|
||||
|
||||
expect(loadChatHistoryMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("addExecApproval", () => {
|
||||
it("keeps the newest approval at the front of the queue", () => {
|
||||
const queue = addExecApproval(
|
||||
|
||||
@@ -393,6 +393,17 @@ function handleChatGatewayEvent(host: GatewayHost, payload: ChatEventPayload | u
|
||||
}
|
||||
}
|
||||
|
||||
function handleSessionMessageGatewayEvent(
|
||||
host: GatewayHost,
|
||||
payload: { sessionKey?: string } | undefined,
|
||||
) {
|
||||
const sessionKey = payload?.sessionKey?.trim();
|
||||
if (!sessionKey || sessionKey !== host.sessionKey) {
|
||||
return;
|
||||
}
|
||||
void loadChatHistory(host as unknown as ChatState);
|
||||
}
|
||||
|
||||
function handleGatewayEventUnsafe(host: GatewayHost, evt: GatewayEventFrame) {
|
||||
host.eventLogBuffer = [
|
||||
{ ts: Date.now(), event: evt.event, payload: evt.payload },
|
||||
@@ -429,6 +440,11 @@ function handleGatewayEventUnsafe(host: GatewayHost, evt: GatewayEventFrame) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (evt.event === "session.message") {
|
||||
handleSessionMessageGatewayEvent(host, evt.payload as { sessionKey?: string } | undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
if (evt.event === "presence") {
|
||||
const payload = evt.payload as { presence?: PresenceEntry[] } | undefined;
|
||||
if (payload?.presence && Array.isArray(payload.presence)) {
|
||||
|
||||
Reference in New Issue
Block a user