vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 06:20:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: stabilize live media and gateway probes

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-21 02:09:58 +01:00
parent 5ab26a8774
commit f04185cc70
7 changed files with 75 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/.generated/plugin-sdk-api-baseline.sha256
View File

@@ -1,2 +1,2 @@
75857675ae94b675f1398444330a8404a4551f59bf7179deabba9884c194b2f8 plugin-sdk-api-baseline.json
1f237dbd11ba7f1d25f6371cb4c5f78b288bca0b92bc397624faaf514f12ea9f plugin-sdk-api-baseline.jsonl
e6da774a43c16fddc77e04b0d2888d06454d1adb84814c8db4fee0f495c1eec1 plugin-sdk-api-baseline.json
ef8b5fd8081dfa05740f6a609144e755d95a19196a1617037dba1213134699df plugin-sdk-api-baseline.jsonl

5
extensions/openai/openai.live.test.ts
View File

@@ -324,9 +324,10 @@ describeLive("openai plugin live", () => {
fileName: "reference.png",
mime: "image/png",
prompt: "Reply with one lowercase word for the dominant center color.",
timeoutMs: 30_000,
timeoutMs: 60_000,
agentDir,
cfg,
authStore: EMPTY_AUTH_STORE,
model: LIVE_VISION_MODEL,
provider: "openai",
});
@@ -335,5 +336,5 @@ describeLive("openai plugin live", () => {
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
}, 60_000);
}, 120_000);
});

19
src/agents/models.profiles.live.test.ts
View File

@@ -36,6 +36,7 @@ const LIVE_SETUP_TIMEOUT_MS = Math.max(
1_000,
toInt(process.env.OPENCLAW_LIVE_SETUP_TIMEOUT_MS, 45_000),
);
const LIVE_MODELS_JSON_TIMEOUT_MS = resolveLiveModelsJsonTimeoutMs();
const describeLive = LIVE ? describe : describe.skip;
@@ -270,6 +271,23 @@ function toInt(value: string | undefined, fallback: number): number {
return Number.isFinite(parsed) ? parsed : fallback;
}
function resolveLiveModelsJsonTimeoutMs(
modelsJsonTimeoutRaw = process.env.OPENCLAW_LIVE_MODELS_JSON_TIMEOUT_MS,
setupTimeoutMs = LIVE_SETUP_TIMEOUT_MS,
): number {
return Math.max(setupTimeoutMs, toInt(modelsJsonTimeoutRaw, 120_000));
}
describe("resolveLiveModelsJsonTimeoutMs", () => {
it("defaults models.json preparation to a longer setup timeout", () => {
expect(resolveLiveModelsJsonTimeoutMs(undefined, 45_000)).toBe(120_000);
});
it("never goes below the shared live setup timeout", () => {
expect(resolveLiveModelsJsonTimeoutMs("30000", 45_000)).toBe(45_000);
});
});
function resolveTestReasoning(
model: Model<Api>,
): "minimal" | "low" | "medium" | "high" | "xhigh" | undefined {
@@ -427,6 +445,7 @@ describeLive("live models (profile keys)", () => {
await withLiveStageTimeout(
ensureOpenClawModelsJson(cfg),
"[live-models] prepare models.json",
LIVE_MODELS_JSON_TIMEOUT_MS,
);
if (!DIRECT_ENABLED) {
logProgress(

44
src/gateway/gateway-models.profiles.live.test.ts
View File

@@ -318,6 +318,10 @@ function isMeaningful(text: string): boolean {
return true;
}
function hasEventLoopPromptKeywords(text: string): boolean {
return /\bmicro\s*-?\s*tasks?\b/i.test(text) && /\bmacro\s*-?\s*tasks?\b/i.test(text);
}
function shouldStripAssistantScaffoldingForLiveModel(modelKey?: string): boolean {
if (!modelKey) {
return false;
@@ -708,6 +712,19 @@ describe("isPromptProbeMiss", () => {
expect(isPromptProbeMiss(error)).toBe(expected);
});
});
describe("hasEventLoopPromptKeywords", () => {
it.each([
{
text: "The event loop drains the microtask queue before running the next macrotask.",
expected: true,
},
{ text: "Micro-tasks run before macro-tasks.", expected: true },
{ text: "Promise callbacks run before timer callbacks.", expected: false },
])("returns $expected for $text", ({ text, expected }) => {
expect(hasEventLoopPromptKeywords(text)).toBe(expected);
});
});
function isMissingProfileError(error: string): boolean {
return /no credentials found for profile/i.test(error);
}
@@ -1531,6 +1548,28 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) {
phase: "prompt",
label: params.label,
});
if (!isMeaningful(text) || !hasEventLoopPromptKeywords(text)) {
logProgress(`${progressLabel}: prompt retry (weak answer)`);
const retryText = await requestGatewayAgentText({
client,
sessionKey,
idempotencyKey: `idem-${randomUUID()}-keyword-retry`,
modelKey,
message:
"Answer in exactly two short sentences. Include the exact lowercase words microtask and macrotask. No bullets.",
thinkingLevel: params.thinkingLevel,
context: `${progressLabel}: prompt-keyword-retry`,
});
if (retryText) {
text = retryText;
assertNoReasoningTags({
text,
model: modelKey,
phase: "prompt-retry",
label: params.label,
});
}
}
if (!isMeaningful(text)) {
if (isGoogleishProvider(model.provider) && /gemini/i.test(model.id)) {
logProgress(`${progressLabel}: skip (google not meaningful)`);
@@ -1538,10 +1577,7 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) {
}
throw new Error(`not meaningful: ${text}`);
}
if (
!/\bmicro\s*-?\s*tasks?\b/i.test(text) ||
!/\bmacro\s*-?\s*tasks?\b/i.test(text)
) {
if (!hasEventLoopPromptKeywords(text)) {
throw new Error(`missing required keywords: ${text}`);
}

6
src/media-understanding/image.test.ts
View File

@@ -90,6 +90,7 @@ describe("describeImageWithModel", () => {
});
it("routes minimax-portal image models through the MiniMax VLM endpoint", async () => {
const authStore = { version: 1, profiles: {} };
const result = await describeImageWithModel({
cfg: {},
agentDir: "/tmp/openclaw-agent",
@@ -100,6 +101,7 @@ describe("describeImageWithModel", () => {
mime: "image/png",
prompt: "Describe the image.",
timeoutMs: 1000,
authStore,
});
expect(result).toEqual({
@@ -107,7 +109,9 @@ describe("describeImageWithModel", () => {
model: "MiniMax-VL-01",
});
expect(ensureOpenClawModelsJsonMock).toHaveBeenCalled();
expect(getApiKeyForModelMock).toHaveBeenCalled();
expect(getApiKeyForModelMock).toHaveBeenCalledWith(
expect.objectContaining({ store: authStore }),
);
expect(requireApiKeyMock).toHaveBeenCalled();
expect(setRuntimeApiKeyMock).toHaveBeenCalledWith("minimax-portal", "oauth-test");
expect(fetchMock).toHaveBeenCalledWith(

3
src/media-understanding/image.ts
View File

@@ -43,6 +43,7 @@ async function resolveImageRuntime(params: {
model: string;
profile?: string;
preferredProfile?: string;
authStore?: ImageDescriptionRequest["authStore"];
}): Promise<{ apiKey: string; model: Model<Api> }> {
await ensureOpenClawModelsJson(params.cfg, params.agentDir);
const { discoverAuthStorage, discoverModels } = await loadPiModelDiscoveryRuntime();
@@ -62,6 +63,7 @@ async function resolveImageRuntime(params: {
agentDir: params.agentDir,
profileId: params.profile,
preferredProfile: params.preferredProfile,
store: params.authStore,
});
const apiKey = requireApiKey(apiKeyInfo, model.provider);
authStorage.setRuntimeApiKey(model.provider, apiKey);
@@ -226,6 +228,7 @@ export async function describeImageWithModel(
timeoutMs: params.timeoutMs,
profile: params.profile,
preferredProfile: params.preferredProfile,
authStore: params.authStore,
agentDir: params.agentDir,
cfg: params.cfg,
});

3
src/media-understanding/types.ts
View File

@@ -1,3 +1,4 @@
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
export type MediaUnderstandingKind =
@@ -136,6 +137,7 @@ export type ImageDescriptionRequest = {
timeoutMs: number;
profile?: string;
preferredProfile?: string;
authStore?: AuthProfileStore;
agentDir: string;
cfg: OpenClawConfig;
model: string;
@@ -157,6 +159,7 @@ export type ImagesDescriptionRequest = {
timeoutMs: number;
profile?: string;
preferredProfile?: string;
authStore?: AuthProfileStore;
agentDir: string;
cfg: OpenClawConfig;
};