openclaw/extensions/qa-lab/src/suite-runtime-flow.ts

// Qa Lab plugin module implements suite runtime flow behavior.
import { createHash, randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { formatMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status";
import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-host-core";
import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing";
import { createPluginStateSyncKeyedStore } from "openclaw/plugin-sdk/runtime-doctor";
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
import {
  callQaBrowserRequest,
  qaBrowserAct,
  qaBrowserOpenTab,
  qaBrowserSnapshot,
  waitForQaBrowserReady,
} from "./browser-runtime.js";
import { waitForCronRunCompletion } from "./cron-run-wait.js";
import {
  hasDiscoveryLabels,
  reportsDiscoveryScopeLeak,
  reportsMissingDiscoveryFiles,
} from "./discovery-eval.js";
import { extractQaToolPayload } from "./extract-tool-payload.js";
import { assertNoGatewayLogSentinels, scanGatewayLogSentinels } from "./gateway-log-sentinel.js";
import { hasModelSwitchContinuitySignal } from "./model-switch-eval.js";
import { qaChannelPlugin } from "./runtime-api.js";
import { runRuntimeToolFixture } from "./runtime-tool-fixture.js";
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
import { createQaScenarioRuntimeApi, type QaScenarioRuntimeEnv } from "./scenario-runtime-api.js";
import {
  callPluginToolsMcp,
  createSession,
  ensureImageGenerationConfigured,
  extractMediaPathFromText,
  findSkill,
  forceMemoryIndex,
  findManagedDreamingCronJob,
  handleQaAction,
  listCronJobs,
  readDoctorMemoryStatus,
  readEffectiveTools,
  readRawQaSessionStore,
  readSessionTranscriptSummary,
  readSkillStatus,
  resolveGeneratedImagePath,
  runAgentPrompt,
  runQaCli,
  seedQaSessionTranscript,
  startAgentRun,
  waitForAgentHistoryReply,
  waitForAgentRun,
  writeWorkspaceSkill,
} from "./suite-runtime-agent.js";
import {
  applyConfig,
  fetchJson,
  patchConfig,
  readConfigSnapshot,
  restartGatewayWithConfigPatch,
  waitForConfigRestartSettle,
  waitForGatewayHealthy,
  waitForQaChannelReady,
  waitForTransportReady,
} from "./suite-runtime-gateway.js";
import {
  formatConversationTranscript,
  formatTransportTranscript,
  readTransportTranscript,
  recentOutboundSummary,
  waitForChannelOutboundMessage,
  waitForNoOutbound,
  waitForNoTransportOutbound,
  waitForOutboundMessage,
  waitForTransportOutboundMessage,
} from "./suite-runtime-transport.js";
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
import {
  qaWebEvaluate,
  qaWebOpenPage,
  qaWebSnapshot,
  qaWebType,
  qaWebWait,
} from "./web-runtime.js";

type QaSuiteScenarioFlowEnv = {
  lab: unknown;
  webSessionIds: Set<string>;
  transport: QaSuiteRuntimeEnv["transport"] & QaScenarioRuntimeEnv["transport"];
} & Omit<QaSuiteRuntimeEnv, "transport">;

function activeMemoryToggleKey(sessionKey: string) {
  return createHash("sha256").update(sessionKey, "utf8").digest("hex");
}

function setActiveMemorySessionDisabled(
  env: QaSuiteScenarioFlowEnv,
  sessionKey: string,
  disabled: boolean,
) {
  const store = createPluginStateSyncKeyedStore<{
    sessionKey: string;
    disabled: true;
    updatedAt: number;
  }>("active-memory", {
    namespace: "session-toggles",
    maxEntries: 10_000,
    env: {
      ...process.env,
      OPENCLAW_STATE_DIR: path.join(env.gateway.tempRoot, "state"),
    },
  });
  const key = activeMemoryToggleKey(sessionKey);
  if (disabled) {
    store.register(key, {
      sessionKey,
      disabled: true,
      updatedAt: Date.now(),
    });
    return;
  }
  store.delete(key);
}

type QaSuiteStep = {
  name: string;
  run: () => Promise<string | void>;
};

type QaSuiteScenarioResult = {
  name: string;
  status: "pass" | "fail";
  steps: Array<{
    name: string;
    status: "pass" | "fail" | "skip";
    details?: string;
  }>;
  details?: string;
};

type QaSuiteScenarioDepsParams = {
  env: QaSuiteScenarioFlowEnv;
  runScenario: (name: string, steps: QaSuiteStep[]) => Promise<QaSuiteScenarioResult>;
  splitModelRef: (ref: string) => { provider: string; model: string } | null;
  formatErrorMessage: (error: unknown) => string;
  liveTurnTimeoutMs: (
    env: Pick<QaSuiteRuntimeEnv, "providerMode" | "primaryModel" | "alternateModel">,
    fallbackMs: number,
  ) => number;
  resolveQaLiveTurnTimeoutMs: (
    env: Pick<QaSuiteRuntimeEnv, "providerMode" | "primaryModel" | "alternateModel">,
    fallbackMs: number,
  ) => number;
};

type QaSuiteScenarioFlowApiParams = QaSuiteScenarioDepsParams & {
  scenario: QaSeedScenarioWithSource;
  constants: {
    imageUnderstandingPngBase64: string;
    imageUnderstandingLargePngBase64: string;
    imageUnderstandingValidPngBase64: string;
  };
};

function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) {
  return {
    fs,
    path,
    sleep,
    randomUUID,
    runScenario: params.runScenario,
    waitForOutboundMessage,
    waitForTransportOutboundMessage,
    waitForChannelOutboundMessage,
    waitForNoOutbound,
    waitForNoTransportOutbound,
    recentOutboundSummary,
    formatConversationTranscript,
    readTransportTranscript,
    formatTransportTranscript,
    fetchJson,
    waitForGatewayHealthy,
    waitForTransportReady,
    waitForQaChannelReady,
    browserRequest: callQaBrowserRequest,
    waitForBrowserReady: waitForQaBrowserReady,
    browserOpenTab: qaBrowserOpenTab,
    browserSnapshot: qaBrowserSnapshot,
    browserAct: qaBrowserAct,
    webOpenPage: async (webParams: Parameters<typeof qaWebOpenPage>[0]) => {
      const opened = await qaWebOpenPage({ ...webParams, repoRoot: params.env.repoRoot });
      params.env.webSessionIds.add(opened.pageId);
      return opened;
    },
    webWait: qaWebWait,
    webType: qaWebType,
    webSnapshot: qaWebSnapshot,
    webEvaluate: qaWebEvaluate,
    waitForConfigRestartSettle,
    patchConfig,
    applyConfig,
    readConfigSnapshot,
    restartGatewayWithConfigPatch,
    createSession,
    readEffectiveTools,
    readSkillStatus,
    readRawQaSessionStore,
    seedQaSessionTranscript,
    readGatewayLogs: () => params.env.gateway.logs?.() ?? "",
    markGatewayLogCursor: () => (params.env.gateway.logs?.() ?? "").length,
    scanGatewayLogSentinels: (options?: Parameters<typeof scanGatewayLogSentinels>[1]) =>
      scanGatewayLogSentinels(params.env.gateway.logs?.(), options),
    assertNoGatewayLogSentinels: (options?: Parameters<typeof assertNoGatewayLogSentinels>[1]) =>
      assertNoGatewayLogSentinels(params.env.gateway.logs?.(), options),
    readSessionTranscriptSummary,
    runQaCli,
    extractMediaPathFromText,
    resolveGeneratedImagePath,
    startAgentRun,
    waitForAgentRun,
    waitForAgentHistoryReply,
    listCronJobs,
    findManagedDreamingCronJob,
    waitForCronRunCompletion,
    readDoctorMemoryStatus,
    forceMemoryIndex,
    findSkill,
    writeWorkspaceSkill,
    callPluginToolsMcp,
    runAgentPrompt,
    ensureImageGenerationConfigured,
    handleQaAction,
    runRuntimeToolFixture: async (
      envArg: QaSuiteScenarioFlowEnv,
      configArg: Record<string, unknown>,
    ) =>
      runRuntimeToolFixture(envArg, configArg, {
        createSession,
        readEffectiveTools,
        runAgentPrompt,
        fetchJson,
        ensureImageGenerationConfigured,
      }),
    extractQaToolPayload,
    formatMemoryDreamingDay,
    resolveSessionTranscriptsDirForAgent,
    activeMemoryToggleKey,
    setActiveMemorySessionDisabled,
    buildAgentSessionKey,
    normalizeLowercaseStringOrEmpty,
    formatErrorMessage: params.formatErrorMessage,
    liveTurnTimeoutMs: params.liveTurnTimeoutMs,
    resolveQaLiveTurnTimeoutMs: params.resolveQaLiveTurnTimeoutMs,
    splitModelRef: params.splitModelRef,
    qaChannelPlugin,
    hasDiscoveryLabels,
    reportsDiscoveryScopeLeak,
    reportsMissingDiscoveryFiles,
    hasModelSwitchContinuitySignal,
  };
}

export function createQaSuiteScenarioFlowApi(params: QaSuiteScenarioFlowApiParams) {
  return createQaScenarioRuntimeApi({
    env: params.env,
    scenario: params.scenario,
    deps: createQaSuiteScenarioDeps({
      env: params.env,
      runScenario: params.runScenario,
      splitModelRef: params.splitModelRef,
      formatErrorMessage: params.formatErrorMessage,
      liveTurnTimeoutMs: params.liveTurnTimeoutMs,
      resolveQaLiveTurnTimeoutMs: params.resolveQaLiveTurnTimeoutMs,
    }),
    constants: params.constants,
  });
}