mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:10:44 +00:00
refactor: trim qa lab helper exports
This commit is contained in:
@@ -3,7 +3,7 @@ import {
|
||||
QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
|
||||
} from "./agentic-parity.js";
|
||||
|
||||
export type QaParityReportStep = {
|
||||
type QaParityReportStep = {
|
||||
name: string;
|
||||
status: "pass" | "fail" | "skip";
|
||||
details?: string;
|
||||
@@ -23,7 +23,7 @@ export type QaParityReportScenario = {
|
||||
* skips the label-match verification for backwards compatibility
|
||||
* with legacy summaries that predate the run metadata block.
|
||||
*/
|
||||
export type QaParityRunBlock = {
|
||||
type QaParityRunBlock = {
|
||||
primaryProvider?: string;
|
||||
primaryModel?: string;
|
||||
primaryModelName?: string;
|
||||
@@ -42,7 +42,7 @@ export type QaParitySuiteSummary = {
|
||||
run?: QaParityRunBlock;
|
||||
};
|
||||
|
||||
export type QaAgenticParityMetrics = {
|
||||
type QaAgenticParityMetrics = {
|
||||
totalScenarios: number;
|
||||
passedScenarios: number;
|
||||
failedScenarios: number;
|
||||
@@ -54,7 +54,7 @@ export type QaAgenticParityMetrics = {
|
||||
fakeSuccessCount: number;
|
||||
};
|
||||
|
||||
export type QaAgenticParityScenarioComparison = {
|
||||
type QaAgenticParityScenarioComparison = {
|
||||
name: string;
|
||||
candidateStatus: "pass" | "fail" | "skip" | "missing";
|
||||
baselineStatus: "pass" | "fail" | "skip" | "missing";
|
||||
@@ -62,7 +62,7 @@ export type QaAgenticParityScenarioComparison = {
|
||||
baselineDetails?: string;
|
||||
};
|
||||
|
||||
export type QaAgenticParityComparison = {
|
||||
type QaAgenticParityComparison = {
|
||||
candidateLabel: string;
|
||||
baselineLabel: string;
|
||||
comparedAt: string;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
export const QA_AGENTIC_PARITY_PACK = "agentic";
|
||||
const QA_AGENTIC_PARITY_PACK = "agentic";
|
||||
|
||||
export const QA_AGENTIC_PARITY_SCENARIOS = [
|
||||
const QA_AGENTIC_PARITY_SCENARIOS = [
|
||||
{
|
||||
id: "approval-turn-tool-followthrough",
|
||||
title: "Approval turn tool followthrough",
|
||||
|
||||
@@ -33,7 +33,7 @@ export type QaCharacterModelOptions = {
|
||||
fastMode?: boolean;
|
||||
};
|
||||
|
||||
export type QaCharacterEvalRun = {
|
||||
type QaCharacterEvalRun = {
|
||||
model: string;
|
||||
status: QaCharacterRunStatus;
|
||||
durationMs: number;
|
||||
@@ -61,7 +61,7 @@ export type QaCharacterEvalJudgment = {
|
||||
weaknesses: string[];
|
||||
};
|
||||
|
||||
export type QaCharacterEvalResult = {
|
||||
type QaCharacterEvalResult = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
@@ -69,7 +69,7 @@ export type QaCharacterEvalResult = {
|
||||
judgments: QaCharacterEvalJudgeResult[];
|
||||
};
|
||||
|
||||
export type QaCharacterEvalJudgeResult = {
|
||||
type QaCharacterEvalJudgeResult = {
|
||||
model: string;
|
||||
thinkingDefault: QaThinkingLevel;
|
||||
fastMode: boolean;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
|
||||
export type QaCoverageScenarioSummary = {
|
||||
type QaCoverageScenarioSummary = {
|
||||
id: string;
|
||||
title: string;
|
||||
sourcePath: string;
|
||||
@@ -9,18 +9,18 @@ export type QaCoverageScenarioSummary = {
|
||||
risk: string;
|
||||
};
|
||||
|
||||
export type QaCoverageIntent = "primary" | "secondary";
|
||||
type QaCoverageIntent = "primary" | "secondary";
|
||||
|
||||
export type QaCoverageScenarioReference = QaCoverageScenarioSummary & {
|
||||
type QaCoverageScenarioReference = QaCoverageScenarioSummary & {
|
||||
intent: QaCoverageIntent;
|
||||
};
|
||||
|
||||
export type QaCoverageFeatureSummary = {
|
||||
type QaCoverageFeatureSummary = {
|
||||
id: string;
|
||||
scenarios: QaCoverageScenarioReference[];
|
||||
};
|
||||
|
||||
export type QaCoverageInventory = {
|
||||
type QaCoverageInventory = {
|
||||
scenarioCount: number;
|
||||
coverageIdCount: number;
|
||||
primaryCoverageIdCount: number;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
export type QaCronRunLogEntry = {
|
||||
type QaCronRunLogEntry = {
|
||||
ts?: number;
|
||||
status?: "ok" | "error" | "skipped";
|
||||
summary?: string;
|
||||
|
||||
@@ -26,7 +26,7 @@ export async function fetchHealthUrl(url: string): Promise<{ ok: boolean }> {
|
||||
}
|
||||
}
|
||||
|
||||
export function describeError(error: unknown) {
|
||||
function describeError(error: unknown) {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ type QaGatewayRpcRequestOptions = {
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
export type QaGatewayRpcClient = {
|
||||
type QaGatewayRpcClient = {
|
||||
request(method: string, rpcParams?: unknown, opts?: QaGatewayRpcRequestOptions): Promise<unknown>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
@@ -10,7 +10,7 @@ const CAPTURE_QUERY_PRESETS = new Set([
|
||||
"error-bursts",
|
||||
]);
|
||||
|
||||
export type QaStartupProbeStatus = {
|
||||
type QaStartupProbeStatus = {
|
||||
label: string;
|
||||
url: string;
|
||||
ok: boolean;
|
||||
|
||||
@@ -50,7 +50,7 @@ export function missingUiHtml() {
|
||||
</html>`;
|
||||
}
|
||||
|
||||
export function resolveUiDistDir(overrideDir?: string | null, repoRoot = process.cwd()) {
|
||||
function resolveUiDistDir(overrideDir?: string | null, repoRoot = process.cwd()) {
|
||||
if (overrideDir?.trim()) {
|
||||
return overrideDir;
|
||||
}
|
||||
|
||||
@@ -7,9 +7,9 @@ export type QaLabLatestReport = {
|
||||
generatedAt: string;
|
||||
};
|
||||
|
||||
export type QaLabRunStatus = "idle" | "running" | "completed";
|
||||
type QaLabRunStatus = "idle" | "running" | "completed";
|
||||
|
||||
export type QaLabScenarioStep = {
|
||||
type QaLabScenarioStep = {
|
||||
name: string;
|
||||
status: "pass" | "fail" | "skip";
|
||||
details?: string;
|
||||
|
||||
@@ -35,7 +35,7 @@ function createQaRunnerCliRegistration(
|
||||
});
|
||||
}
|
||||
|
||||
export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
|
||||
const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
|
||||
telegramQaCliRegistration,
|
||||
discordQaCliRegistration,
|
||||
];
|
||||
|
||||
@@ -113,7 +113,7 @@ type DiscordQaScenarioResult = {
|
||||
details: string;
|
||||
};
|
||||
|
||||
export type DiscordQaRunResult = {
|
||||
type DiscordQaRunResult = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
@@ -197,7 +197,7 @@ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [
|
||||
},
|
||||
];
|
||||
|
||||
export const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
scenarios: DISCORD_QA_SCENARIOS,
|
||||
});
|
||||
|
||||
@@ -232,9 +232,7 @@ function isTruthyOptIn(value: string | undefined) {
|
||||
return normalized === "1" || normalized === "true" || normalized === "yes";
|
||||
}
|
||||
|
||||
export function resolveDiscordQaRuntimeEnv(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): DiscordQaRuntimeEnv {
|
||||
function resolveDiscordQaRuntimeEnv(env: NodeJS.ProcessEnv = process.env): DiscordQaRuntimeEnv {
|
||||
const runtimeEnv = {
|
||||
guildId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_GUILD_ID"),
|
||||
channelId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_CHANNEL_ID"),
|
||||
|
||||
@@ -51,7 +51,7 @@ type ConvexCredentialBrokerConfig = {
|
||||
role: QaCredentialRole;
|
||||
};
|
||||
|
||||
export type QaCredentialLeaseHeartbeat = {
|
||||
type QaCredentialLeaseHeartbeat = {
|
||||
getFailure(): Error | null;
|
||||
stop(): Promise<void>;
|
||||
throwIfFailed(): void;
|
||||
@@ -59,9 +59,9 @@ export type QaCredentialLeaseHeartbeat = {
|
||||
|
||||
export type QaCredentialRole = "ci" | "maintainer";
|
||||
|
||||
export type QaCredentialLeaseSource = "convex" | "env";
|
||||
type QaCredentialLeaseSource = "convex" | "env";
|
||||
|
||||
export type QaCredentialLease<TPayload> = {
|
||||
type QaCredentialLease<TPayload> = {
|
||||
credentialId?: string;
|
||||
heartbeat(): Promise<void>;
|
||||
heartbeatIntervalMs: number;
|
||||
@@ -75,7 +75,7 @@ export type QaCredentialLease<TPayload> = {
|
||||
source: QaCredentialLeaseSource;
|
||||
};
|
||||
|
||||
export type AcquireQaCredentialLeaseOptions<TPayload> = {
|
||||
type AcquireQaCredentialLeaseOptions<TPayload> = {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
fetchImpl?: typeof fetch;
|
||||
kind: string;
|
||||
|
||||
@@ -36,7 +36,7 @@ export type LiveTransportQaCliRegistration = {
|
||||
register(qa: Command): void;
|
||||
};
|
||||
|
||||
export type LiveTransportQaCredentialCliOptions = {
|
||||
type LiveTransportQaCredentialCliOptions = {
|
||||
sourceDescription?: string;
|
||||
roleDescription?: string;
|
||||
};
|
||||
@@ -49,7 +49,7 @@ export function createLazyCliRuntimeLoader<T>(load: () => Promise<T>) {
|
||||
};
|
||||
}
|
||||
|
||||
export function mapLiveTransportQaCommanderOptions(
|
||||
function mapLiveTransportQaCommanderOptions(
|
||||
opts: LiveTransportQaCommanderOptions,
|
||||
): LiveTransportQaCommandOptions {
|
||||
return {
|
||||
@@ -67,7 +67,7 @@ export function mapLiveTransportQaCommanderOptions(
|
||||
};
|
||||
}
|
||||
|
||||
export function registerLiveTransportQaCli(params: {
|
||||
function registerLiveTransportQaCli(params: {
|
||||
qa: Command;
|
||||
commandName: string;
|
||||
credentialOptions?: LiveTransportQaCredentialCliOptions;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export type LiveTransportStandardScenarioId =
|
||||
type LiveTransportStandardScenarioId =
|
||||
| "canary"
|
||||
| "mention-gating"
|
||||
| "allowlist-block"
|
||||
@@ -16,60 +16,59 @@ export type LiveTransportScenarioDefinition<TId extends string = string> = {
|
||||
title: string;
|
||||
};
|
||||
|
||||
export type LiveTransportStandardScenarioDefinition = {
|
||||
type LiveTransportStandardScenarioDefinition = {
|
||||
description: string;
|
||||
id: LiveTransportStandardScenarioId;
|
||||
title: string;
|
||||
};
|
||||
|
||||
export const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] =
|
||||
[
|
||||
{
|
||||
id: "canary",
|
||||
title: "Transport canary",
|
||||
description: "The lane can trigger one known-good reply on the real transport.",
|
||||
},
|
||||
{
|
||||
id: "mention-gating",
|
||||
title: "Mention gating",
|
||||
description: "Messages without the required mention do not trigger a reply.",
|
||||
},
|
||||
{
|
||||
id: "allowlist-block",
|
||||
title: "Sender allowlist block",
|
||||
description: "Non-allowlisted senders do not trigger a reply.",
|
||||
},
|
||||
{
|
||||
id: "top-level-reply-shape",
|
||||
title: "Top-level reply shape",
|
||||
description: "Top-level replies stay top-level when the lane is configured that way.",
|
||||
},
|
||||
{
|
||||
id: "restart-resume",
|
||||
title: "Restart resume",
|
||||
description: "The lane still responds after a gateway restart.",
|
||||
},
|
||||
{
|
||||
id: "thread-follow-up",
|
||||
title: "Thread follow-up",
|
||||
description: "Threaded prompts receive threaded replies with the expected relation metadata.",
|
||||
},
|
||||
{
|
||||
id: "thread-isolation",
|
||||
title: "Thread isolation",
|
||||
description: "Fresh top-level prompts stay out of prior threads.",
|
||||
},
|
||||
{
|
||||
id: "reaction-observation",
|
||||
title: "Reaction observation",
|
||||
description: "Reaction events are observed and normalized correctly.",
|
||||
},
|
||||
{
|
||||
id: "help-command",
|
||||
title: "Help command",
|
||||
description: "The transport-specific help command path replies successfully.",
|
||||
},
|
||||
] as const;
|
||||
const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [
|
||||
{
|
||||
id: "canary",
|
||||
title: "Transport canary",
|
||||
description: "The lane can trigger one known-good reply on the real transport.",
|
||||
},
|
||||
{
|
||||
id: "mention-gating",
|
||||
title: "Mention gating",
|
||||
description: "Messages without the required mention do not trigger a reply.",
|
||||
},
|
||||
{
|
||||
id: "allowlist-block",
|
||||
title: "Sender allowlist block",
|
||||
description: "Non-allowlisted senders do not trigger a reply.",
|
||||
},
|
||||
{
|
||||
id: "top-level-reply-shape",
|
||||
title: "Top-level reply shape",
|
||||
description: "Top-level replies stay top-level when the lane is configured that way.",
|
||||
},
|
||||
{
|
||||
id: "restart-resume",
|
||||
title: "Restart resume",
|
||||
description: "The lane still responds after a gateway restart.",
|
||||
},
|
||||
{
|
||||
id: "thread-follow-up",
|
||||
title: "Thread follow-up",
|
||||
description: "Threaded prompts receive threaded replies with the expected relation metadata.",
|
||||
},
|
||||
{
|
||||
id: "thread-isolation",
|
||||
title: "Thread isolation",
|
||||
description: "Fresh top-level prompts stay out of prior threads.",
|
||||
},
|
||||
{
|
||||
id: "reaction-observation",
|
||||
title: "Reaction observation",
|
||||
description: "Reaction events are observed and normalized correctly.",
|
||||
},
|
||||
{
|
||||
id: "help-command",
|
||||
title: "Help command",
|
||||
description: "The transport-specific help command path replies successfully.",
|
||||
},
|
||||
] as const;
|
||||
|
||||
export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] =
|
||||
[
|
||||
|
||||
@@ -117,7 +117,7 @@ type TelegramQaScenarioResult = {
|
||||
|
||||
type TelegramQaCanaryPhase = "sut_reply_timeout" | "sut_reply_not_threaded" | "sut_reply_empty";
|
||||
|
||||
export type TelegramQaRunResult = {
|
||||
type TelegramQaRunResult = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
@@ -298,7 +298,7 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [
|
||||
},
|
||||
];
|
||||
|
||||
export const TELEGRAM_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
const TELEGRAM_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
alwaysOnStandardScenarioIds: ["canary"],
|
||||
scenarios: TELEGRAM_QA_SCENARIOS,
|
||||
});
|
||||
@@ -416,9 +416,7 @@ function formatTelegramQaProgressDetails(details: string): string {
|
||||
return `${sanitized.slice(0, TELEGRAM_QA_PROGRESS_DETAIL_LIMIT - 3).trimEnd()}...`;
|
||||
}
|
||||
|
||||
export function resolveTelegramQaRuntimeEnv(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): TelegramQaRuntimeEnv {
|
||||
function resolveTelegramQaRuntimeEnv(env: NodeJS.ProcessEnv = process.env): TelegramQaRuntimeEnv {
|
||||
const groupId = resolveEnvValue(env, "OPENCLAW_QA_TELEGRAM_GROUP_ID");
|
||||
if (!/^-?\d+$/u.test(groupId)) {
|
||||
throw new Error("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id.");
|
||||
@@ -472,9 +470,7 @@ function detectMediaKinds(message: TelegramMessage) {
|
||||
return kinds;
|
||||
}
|
||||
|
||||
export function normalizeTelegramObservedMessage(
|
||||
update: TelegramUpdate,
|
||||
): TelegramObservedMessage | null {
|
||||
function normalizeTelegramObservedMessage(update: TelegramUpdate): TelegramObservedMessage | null {
|
||||
const message = update.message;
|
||||
if (!message?.from?.id) {
|
||||
return null;
|
||||
|
||||
@@ -32,7 +32,7 @@ const MULTIPASS_REPO_SYNC_EXCLUDES = [
|
||||
const MULTIPASS_EXEC_MAX_BUFFER = 64 * 1024 * 1024;
|
||||
const MULTIPASS_GUEST_RUN_TIMEOUT_MS = 60 * 60 * 1000;
|
||||
|
||||
export const qaMultipassDefaultResources = {
|
||||
const qaMultipassDefaultResources = {
|
||||
image: "lts",
|
||||
cpus: 2,
|
||||
memory: "4G",
|
||||
@@ -52,7 +52,7 @@ type ExecFileOptions = {
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
export type QaMultipassPlan = {
|
||||
type QaMultipassPlan = {
|
||||
repoRoot: string;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
@@ -86,7 +86,7 @@ export type QaMultipassPlan = {
|
||||
qaCommand: string[];
|
||||
};
|
||||
|
||||
export type QaMultipassRunResult = {
|
||||
type QaMultipassRunResult = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
|
||||
@@ -20,8 +20,8 @@ const QA_LIVE_ENV_ALIASES = Object.freeze([
|
||||
]);
|
||||
|
||||
export const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH";
|
||||
export const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
|
||||
export const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
|
||||
const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
|
||||
const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
|
||||
export type QaCliBackendAuthMode = "auto" | "api-key" | "subscription";
|
||||
|
||||
export const QA_PROVIDER_SECRET_ENV_VARS = Object.freeze([
|
||||
|
||||
@@ -3,21 +3,16 @@ import { liveFrontierProviderDefinition } from "./live-frontier/index.js";
|
||||
import { mockOpenAiProviderDefinition } from "./mock-openai/index.js";
|
||||
import type { QaProviderDefinition, QaProviderMode, QaProviderModeInput } from "./shared/types.js";
|
||||
|
||||
export type {
|
||||
QaMockProviderServer,
|
||||
QaProviderDefinition,
|
||||
QaProviderMode,
|
||||
QaProviderModeInput,
|
||||
} from "./shared/types.js";
|
||||
export type { QaMockProviderServer, QaProviderMode, QaProviderModeInput } from "./shared/types.js";
|
||||
|
||||
const PROVIDERS = [
|
||||
const PROVIDERS: readonly QaProviderDefinition[] = [
|
||||
mockOpenAiProviderDefinition,
|
||||
aimockProviderDefinition,
|
||||
liveFrontierProviderDefinition,
|
||||
] as const satisfies readonly QaProviderDefinition[];
|
||||
] as const;
|
||||
|
||||
export const DEFAULT_QA_PROVIDER_MODE = "mock-openai" satisfies QaProviderMode;
|
||||
export const DEFAULT_QA_LIVE_PROVIDER_MODE = "live-frontier" satisfies QaProviderMode;
|
||||
export const DEFAULT_QA_PROVIDER_MODE: QaProviderMode = "mock-openai";
|
||||
export const DEFAULT_QA_LIVE_PROVIDER_MODE: QaProviderMode = "live-frontier";
|
||||
|
||||
const PROVIDERS_BY_INPUT = new Map<QaProviderModeInput, QaProviderDefinition>();
|
||||
for (const provider of PROVIDERS) {
|
||||
@@ -40,7 +35,7 @@ export function getQaProvider(input: QaProviderModeInput): QaProviderDefinition
|
||||
return provider;
|
||||
}
|
||||
|
||||
export function listQaProviderModes() {
|
||||
function listQaProviderModes() {
|
||||
return PROVIDERS.map((provider) => provider.mode);
|
||||
}
|
||||
|
||||
|
||||
@@ -3,12 +3,12 @@ import { applyAuthProfileConfig } from "openclaw/plugin-sdk/provider-auth-api-ke
|
||||
import { resolveQaAgentAuthDir, writeQaAuthProfiles } from "./auth-store.js";
|
||||
|
||||
/** Providers the mock harness stages placeholder credentials for by default. */
|
||||
export const QA_MOCK_AUTH_PROVIDERS = Object.freeze(["openai", "anthropic"] as const);
|
||||
const QA_MOCK_AUTH_PROVIDERS = Object.freeze(["openai", "anthropic"] as const);
|
||||
|
||||
/** Agent IDs the mock harness stages credentials under. */
|
||||
export const QA_MOCK_AUTH_AGENT_IDS = Object.freeze(["main", "qa"] as const);
|
||||
const QA_MOCK_AUTH_AGENT_IDS = Object.freeze(["main", "qa"] as const);
|
||||
|
||||
export function buildQaMockProfileId(provider: string): string {
|
||||
function buildQaMockProfileId(provider: string): string {
|
||||
return `qa-mock-${provider}`;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,11 +14,11 @@ function cloneProvider(provider: ModelProviderConfig): ModelProviderConfig {
|
||||
};
|
||||
}
|
||||
|
||||
export function trimTrailingApiV1(baseUrl: string) {
|
||||
function trimTrailingApiV1(baseUrl: string) {
|
||||
return baseUrl.replace(/\/v1\/?$/i, "");
|
||||
}
|
||||
|
||||
export function createMockOpenAiResponsesProvider(baseUrl: string): ModelProviderConfig {
|
||||
function createMockOpenAiResponsesProvider(baseUrl: string): ModelProviderConfig {
|
||||
return {
|
||||
baseUrl,
|
||||
apiKey: "test",
|
||||
@@ -61,7 +61,7 @@ export function createMockOpenAiResponsesProvider(baseUrl: string): ModelProvide
|
||||
};
|
||||
}
|
||||
|
||||
export function createMockAnthropicMessagesProvider(baseUrl: string): ModelProviderConfig {
|
||||
function createMockAnthropicMessagesProvider(baseUrl: string): ModelProviderConfig {
|
||||
return {
|
||||
baseUrl: trimTrailingApiV1(baseUrl),
|
||||
apiKey: "test",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createMockProviderMap } from "./mock-model-config.js";
|
||||
import type { QaProviderDefinition, QaProviderMode } from "./types.js";
|
||||
|
||||
export type MockQaProviderDefinitionParams = {
|
||||
type MockQaProviderDefinitionParams = {
|
||||
mode: Extract<QaProviderMode, "aimock" | "mock-openai">;
|
||||
commandName: string;
|
||||
commandDescription: string;
|
||||
|
||||
@@ -9,22 +9,22 @@ export type QaMockProviderServer = {
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
export type QaProviderModelParamsInput = {
|
||||
type QaProviderModelParamsInput = {
|
||||
modelRef: string;
|
||||
fastMode?: boolean;
|
||||
thinkingDefault?: QaThinkingLevel;
|
||||
};
|
||||
|
||||
export type QaProviderGatewayModelsInput = {
|
||||
type QaProviderGatewayModelsInput = {
|
||||
providerBaseUrl: string;
|
||||
liveProviderConfigs?: Record<string, ModelProviderConfig>;
|
||||
};
|
||||
|
||||
export type QaProviderDefaultImageInput = {
|
||||
type QaProviderDefaultImageInput = {
|
||||
modelProviderIds: readonly string[];
|
||||
};
|
||||
|
||||
export type QaProviderTurnTimeoutInput = {
|
||||
type QaProviderTurnTimeoutInput = {
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
modelRef: string;
|
||||
|
||||
@@ -12,8 +12,8 @@ import type {
|
||||
} from "./qa-transport.js";
|
||||
import { qaChannelPlugin } from "./runtime-api.js";
|
||||
|
||||
export const QA_CHANNEL_ID = "qa-channel";
|
||||
export const QA_CHANNEL_ACCOUNT_ID = "default";
|
||||
const QA_CHANNEL_ID = "qa-channel";
|
||||
const QA_CHANNEL_ACCOUNT_ID = "default";
|
||||
export const QA_CHANNEL_REQUIRED_PLUGIN_IDS = Object.freeze([QA_CHANNEL_ID]);
|
||||
export const QA_CHANNEL_DEFAULT_SUITE_CONCURRENCY = 4;
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ const listCredentialsResponseSchema = z.object({
|
||||
count: z.number().int().nonnegative().optional(),
|
||||
});
|
||||
|
||||
export type QaCredentialAdminListStatus = z.infer<typeof listStatusSchema>;
|
||||
type QaCredentialAdminListStatus = z.infer<typeof listStatusSchema>;
|
||||
export type QaCredentialRecord = z.infer<typeof credentialRecordSchema>;
|
||||
|
||||
export class QaCredentialAdminError extends Error {
|
||||
@@ -111,13 +111,13 @@ type ListQaCredentialSetsOptions = AdminBaseOptions & {
|
||||
status?: string;
|
||||
};
|
||||
|
||||
export type QaCredentialDoctorCheck = {
|
||||
type QaCredentialDoctorCheck = {
|
||||
details?: string;
|
||||
name: string;
|
||||
status: "fail" | "pass" | "warn";
|
||||
};
|
||||
|
||||
export type QaCredentialDoctorResult = {
|
||||
type QaCredentialDoctorResult = {
|
||||
checks: QaCredentialDoctorCheck[];
|
||||
status: "fail" | "pass" | "warn";
|
||||
};
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
export const QA_CREDENTIALS_DEFAULT_ENDPOINT_PREFIX = "/qa-credentials/v1";
|
||||
export const QA_CREDENTIALS_ALLOW_INSECURE_HTTP_ENV_KEY = "OPENCLAW_QA_ALLOW_INSECURE_HTTP";
|
||||
const QA_CREDENTIALS_ALLOW_INSECURE_HTTP_ENV_KEY = "OPENCLAW_QA_ALLOW_INSECURE_HTTP";
|
||||
|
||||
type ErrorFactory = (message: string) => Error;
|
||||
|
||||
|
||||
@@ -46,14 +46,14 @@ export type QaTransportState = {
|
||||
waitFor: (input: QaBusWaitForInput) => Promise<unknown>;
|
||||
};
|
||||
|
||||
export type QaTransportFailureCursorSpace = "all" | "outbound";
|
||||
type QaTransportFailureCursorSpace = "all" | "outbound";
|
||||
|
||||
export type QaTransportFailureAssertionOptions = {
|
||||
type QaTransportFailureAssertionOptions = {
|
||||
sinceIndex?: number;
|
||||
cursorSpace?: QaTransportFailureCursorSpace;
|
||||
};
|
||||
|
||||
export type QaTransportCommonCapabilities = {
|
||||
type QaTransportCommonCapabilities = {
|
||||
sendInboundMessage: QaTransportState["addInboundMessage"];
|
||||
injectOutboundMessage: QaTransportState["addOutboundMessage"];
|
||||
waitForOutboundMessage: (input: QaBusWaitForInput) => Promise<unknown>;
|
||||
@@ -113,7 +113,7 @@ export function findFailureOutboundMessage(
|
||||
);
|
||||
}
|
||||
|
||||
export function assertNoFailureReplies(
|
||||
function assertNoFailureReplies(
|
||||
state: QaTransportState,
|
||||
options?: QaTransportFailureAssertionOptions,
|
||||
) {
|
||||
|
||||
@@ -13,7 +13,7 @@ import type { QaSeedScenario } from "./scenario-catalog.js";
|
||||
export type { QaProviderMode } from "./model-selection.js";
|
||||
export type { QaProviderModeInput } from "./providers/index.js";
|
||||
|
||||
export type QaLabRunSelection = {
|
||||
type QaLabRunSelection = {
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
@@ -21,14 +21,14 @@ export type QaLabRunSelection = {
|
||||
scenarioIds: string[];
|
||||
};
|
||||
|
||||
export type QaLabRunArtifacts = {
|
||||
type QaLabRunArtifacts = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
watchUrl: string;
|
||||
};
|
||||
|
||||
export type QaLabRunnerSnapshot = {
|
||||
type QaLabRunnerSnapshot = {
|
||||
status: "idle" | "running" | "completed" | "failed";
|
||||
selection: QaLabRunSelection;
|
||||
startedAt?: string;
|
||||
|
||||
@@ -98,7 +98,7 @@ export type QaScenarioRuntimeConstants = {
|
||||
imageUnderstandingValidPngBase64: string;
|
||||
};
|
||||
|
||||
export type QaScenarioRuntimeApi<
|
||||
type QaScenarioRuntimeApi<
|
||||
TEnv extends QaScenarioRuntimeEnv = QaScenarioRuntimeEnv,
|
||||
TDeps extends QaScenarioRuntimeDeps = QaScenarioRuntimeDeps,
|
||||
> = {
|
||||
|
||||
@@ -4,7 +4,6 @@ import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "./cli-pa
|
||||
import type { QaCliBackendAuthMode } from "./gateway-child.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
import type { QaTransportId } from "./qa-transport-registry.js";
|
||||
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
|
||||
|
||||
const DEFAULT_QA_SUITE_CONCURRENCY = 64;
|
||||
@@ -270,10 +269,7 @@ export {
|
||||
normalizeQaSuiteConcurrency,
|
||||
resolveQaSuiteWorkerStartStaggerMs,
|
||||
resolveQaSuiteOutputDir,
|
||||
scenarioMatchesLiveLane,
|
||||
scenarioRequiresControlUi,
|
||||
selectQaSuiteScenarios,
|
||||
splitModelRef,
|
||||
};
|
||||
|
||||
export type { QaTransportId };
|
||||
|
||||
@@ -12,4 +12,3 @@ function liveTurnTimeoutMs(env: QaLiveTimeoutEnv, fallbackMs: number) {
|
||||
}
|
||||
|
||||
export { liveTurnTimeoutMs };
|
||||
export type { QaLiveTimeoutEnv };
|
||||
|
||||
@@ -357,15 +357,12 @@ async function applyConfig(params: {
|
||||
export {
|
||||
applyConfig,
|
||||
fetchJson,
|
||||
formatGatewayPrimaryErrorText,
|
||||
getGatewayRetryAfterMs,
|
||||
isConfigApplyNoopForSnapshot,
|
||||
isConfigPatchNoopForSnapshot,
|
||||
isConfigHashConflict,
|
||||
isGatewayRestartRace,
|
||||
patchConfig,
|
||||
readConfigSnapshot,
|
||||
runConfigMutation,
|
||||
waitForConfigRestartSettle,
|
||||
waitForGatewayHealthy,
|
||||
waitForQaChannelReady,
|
||||
|
||||
@@ -19,8 +19,6 @@ function createScenarioWaitForCondition(state: QaTransportState) {
|
||||
return createFailureAwareTransportWaitForCondition(state);
|
||||
}
|
||||
|
||||
const waitForCondition = createScenarioWaitForCondition;
|
||||
|
||||
async function waitForOutboundMessage(
|
||||
state: QaTransportState,
|
||||
predicate: (message: QaBusMessage) => boolean,
|
||||
@@ -154,7 +152,6 @@ export {
|
||||
readTransportTranscript,
|
||||
recentOutboundSummary,
|
||||
waitForChannelOutboundMessage,
|
||||
waitForCondition,
|
||||
waitForNoOutbound,
|
||||
waitForNoTransportOutbound,
|
||||
waitForOutboundMessage,
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import type { QaTransportActionName, QaTransportAdapter } from "./qa-transport.js";
|
||||
|
||||
export type QaRuntimeGatewayClient = {
|
||||
type QaRuntimeGatewayClient = {
|
||||
baseUrl: string;
|
||||
tempRoot: string;
|
||||
workspaceDir: string;
|
||||
@@ -27,7 +27,7 @@ export type QaRuntimeGatewayClient = {
|
||||
) => Promise<unknown>;
|
||||
};
|
||||
|
||||
export type QaRuntimeTransport = QaTransportAdapter;
|
||||
type QaRuntimeTransport = QaTransportAdapter;
|
||||
|
||||
export type QaSuiteRuntimeEnv = {
|
||||
gateway: QaRuntimeGatewayClient;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
|
||||
export type QaSuiteSummaryScenario = {
|
||||
type QaSuiteSummaryScenario = {
|
||||
name: string;
|
||||
status: "pass" | "fail";
|
||||
steps: unknown[];
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
/* ===== Shared types (unchanged from the bus protocol) ===== */
|
||||
|
||||
export type Conversation = {
|
||||
type Conversation = {
|
||||
id: string;
|
||||
kind: "direct" | "channel";
|
||||
title?: string;
|
||||
};
|
||||
|
||||
export type Attachment = {
|
||||
type Attachment = {
|
||||
id: string;
|
||||
kind: "image" | "video" | "audio" | "file";
|
||||
mimeType: string;
|
||||
@@ -21,13 +21,13 @@ export type Attachment = {
|
||||
transcript?: string;
|
||||
};
|
||||
|
||||
export type Thread = {
|
||||
type Thread = {
|
||||
id: string;
|
||||
conversationId: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
export type Message = {
|
||||
type Message = {
|
||||
id: string;
|
||||
direction: "inbound" | "outbound";
|
||||
conversation: Conversation;
|
||||
@@ -43,7 +43,7 @@ export type Message = {
|
||||
reactions: Array<{ emoji: string; senderId: string }>;
|
||||
};
|
||||
|
||||
export type BusEvent =
|
||||
type BusEvent =
|
||||
| { cursor: number; kind: "thread-created"; thread: Thread }
|
||||
| { cursor: number; kind: string; message?: Message; emoji?: string };
|
||||
|
||||
@@ -62,7 +62,7 @@ export type ReportEnvelope = {
|
||||
};
|
||||
};
|
||||
|
||||
export type SeedScenario = {
|
||||
type SeedScenario = {
|
||||
id: string;
|
||||
title: string;
|
||||
surface: string;
|
||||
@@ -92,13 +92,13 @@ export type Bootstrap = {
|
||||
};
|
||||
};
|
||||
|
||||
export type ScenarioStep = {
|
||||
type ScenarioStep = {
|
||||
name: string;
|
||||
status: "pass" | "fail" | "skip";
|
||||
details?: string;
|
||||
};
|
||||
|
||||
export type ScenarioOutcome = {
|
||||
type ScenarioOutcome = {
|
||||
id: string;
|
||||
name: string;
|
||||
status: "pending" | "running" | "pass" | "fail" | "skip";
|
||||
@@ -108,7 +108,7 @@ export type ScenarioOutcome = {
|
||||
finishedAt?: string;
|
||||
};
|
||||
|
||||
export type ScenarioRun = {
|
||||
type ScenarioRun = {
|
||||
kind: "suite" | "self-check";
|
||||
status: "idle" | "running" | "completed";
|
||||
startedAt?: string;
|
||||
@@ -132,7 +132,7 @@ export type RunnerSelection = {
|
||||
scenarioIds: string[];
|
||||
};
|
||||
|
||||
export type RunnerSnapshot = {
|
||||
type RunnerSnapshot = {
|
||||
status: "idle" | "running" | "completed" | "failed";
|
||||
selection: RunnerSelection;
|
||||
startedAt?: string;
|
||||
@@ -146,7 +146,7 @@ export type RunnerSnapshot = {
|
||||
error: string | null;
|
||||
};
|
||||
|
||||
export type RunnerModelOption = {
|
||||
type RunnerModelOption = {
|
||||
key: string;
|
||||
name: string;
|
||||
provider: string;
|
||||
@@ -158,7 +158,7 @@ export type OutcomesEnvelope = {
|
||||
run: ScenarioRun | null;
|
||||
};
|
||||
|
||||
export type CaptureSessionSummary = {
|
||||
type CaptureSessionSummary = {
|
||||
id: string;
|
||||
startedAt: number;
|
||||
endedAt?: number;
|
||||
@@ -168,7 +168,7 @@ export type CaptureSessionSummary = {
|
||||
eventCount: number;
|
||||
};
|
||||
|
||||
export type CaptureEventView = {
|
||||
type CaptureEventView = {
|
||||
id?: number;
|
||||
ts: number;
|
||||
protocol: string;
|
||||
@@ -192,7 +192,7 @@ export type CaptureEventView = {
|
||||
captureOrigin?: string;
|
||||
};
|
||||
|
||||
export type CaptureQueryPreset =
|
||||
type CaptureQueryPreset =
|
||||
| "none"
|
||||
| "double-sends"
|
||||
| "retry-storms"
|
||||
@@ -213,12 +213,12 @@ export type CaptureQueryEnvelope = {
|
||||
rows: Array<Record<string, string | number | null>>;
|
||||
};
|
||||
|
||||
export type CaptureObservedDimension = {
|
||||
type CaptureObservedDimension = {
|
||||
value: string;
|
||||
count: number;
|
||||
};
|
||||
|
||||
export type CaptureCoverageSummary = {
|
||||
type CaptureCoverageSummary = {
|
||||
sessionId: string;
|
||||
totalEvents: number;
|
||||
unlabeledEventCount: number;
|
||||
@@ -233,14 +233,14 @@ export type CaptureCoverageEnvelope = {
|
||||
coverage: CaptureCoverageSummary;
|
||||
};
|
||||
|
||||
export type CaptureStartupProbeStatus = {
|
||||
type CaptureStartupProbeStatus = {
|
||||
label: string;
|
||||
url: string;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type CaptureStartupStatus = {
|
||||
type CaptureStartupStatus = {
|
||||
proxy: CaptureStartupProbeStatus;
|
||||
gateway: CaptureStartupProbeStatus;
|
||||
qaLab: CaptureStartupProbeStatus;
|
||||
@@ -350,7 +350,7 @@ export type UiState = {
|
||||
|
||||
/* ===== Helpers ===== */
|
||||
|
||||
export function formatTime(timestamp: number) {
|
||||
function formatTime(timestamp: number) {
|
||||
return new Date(timestamp).toLocaleTimeString([], {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
@@ -930,15 +930,15 @@ const MOCK_MODELS: RunnerModelOption[] = [
|
||||
},
|
||||
];
|
||||
|
||||
export function deriveSelectedConversation(state: UiState): string | null {
|
||||
function deriveSelectedConversation(state: UiState): string | null {
|
||||
return state.selectedConversationId ?? state.snapshot?.conversations[0]?.id ?? null;
|
||||
}
|
||||
|
||||
export function deriveSelectedThread(state: UiState): string | null {
|
||||
function deriveSelectedThread(state: UiState): string | null {
|
||||
return state.selectedThreadId ?? null;
|
||||
}
|
||||
|
||||
export function filteredMessages(state: UiState) {
|
||||
function filteredMessages(state: UiState) {
|
||||
const messages = state.snapshot?.messages ?? [];
|
||||
return messages.filter((message) => {
|
||||
if (state.selectedConversationId && message.conversation.id !== state.selectedConversationId) {
|
||||
|
||||
Reference in New Issue
Block a user