vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 06:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

qa: relax repo-contract artifact matcher

Browse Source
This commit is contained in:
pashpashpash
2026-04-12 22:26:22 -07:00
parent b13844732e
commit eede525970
2 changed files with 18 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
extensions/qa-lab/src/scenario-catalog.test.ts
View File

@@ -149,6 +149,8 @@ describe("qa scenario catalog", () => {
workspaceFiles?: Record<string, string>;
prompt?: string;
expectedReplyAll?: string[];
expectedArtifactAll?: string[];
expectedArtifactAny?: string[];
}
| undefined;
@@ -159,6 +161,8 @@ describe("qa scenario catalog", () => {
);
expect(config?.prompt).toContain("Repo contract followthrough check.");
expect(config?.expectedReplyAll).toEqual(["read:", "wrote:", "status:"]);
expect(config?.expectedArtifactAll).toEqual(["repo contract"]);
expect(config?.expectedArtifactAny).toContain("evidence path");
expect(scenario.title).toBe("Instruction followthrough repo contract");
});

15
qa/scenarios/instruction-followthrough-repo-contract.md
View File

@@ -51,6 +51,12 @@ execution:
- "read:"
- "wrote:"
- "status:"
expectedArtifactAll:
- "repo contract"
expectedArtifactAny:
- "evidence path"
- "agent.md"
- "followthrough"
forbiddenNeedles:
- need permission
- need your approval
@@ -91,9 +97,16 @@ steps:
args:
- lambda:
async: true
expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => null))?.includes('Mission: prove you followed the repo contract.') ? await fs.readFile(artifactPath, 'utf8').catch(() => null) : undefined)"
expr: "((await fs.readFile(artifactPath, 'utf8').catch(() => null))?.trim() ? await fs.readFile(artifactPath, 'utf8').catch(() => null) : undefined)"
- expr: liveTurnTimeoutMs(env, 30000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- set: normalizedArtifact
value:
expr: "normalizeLowercaseStringOrEmpty(artifact)"
- assert:
expr: "config.expectedArtifactAll.every((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle))) && config.expectedArtifactAny.some((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))"
message:
expr: "`repo contract artifact missing expected followthrough signals: ${artifact}`"
- set: expectedReplyAll
value:
expr: config.expectedReplyAll.map(normalizeLowercaseStringOrEmpty)