test(qa): consolidate web search scenario (#104280)

Co-authored-by: Jimmy Puckett <jimmy.puckett@spinen.com>
This commit is contained in:
Peter Steinberger
2026-07-11 00:54:15 -07:00
committed by GitHub
parent 18362758d0
commit dce871540b
3 changed files with 11 additions and 39 deletions

View File

@@ -260,11 +260,9 @@ describe("qa scenario catalog", () => {
"gateway.openai-compatible-apis",
"runtime.hosted-tool-use",
]);
expect(readQaScenarioById("openai-web-search-minimal").coverage?.secondary).toContain(
"runtime.reasoning-and-cache-controls",
);
expect(readQaScenarioById("openai-web-search-native-assertions").coverage?.secondary).toEqual(
expect(readQaScenarioById("openai-web-search-minimal").coverage?.secondary).toEqual(
expect.arrayContaining([
"runtime.reasoning-and-cache-controls",
"web-search.openai-native-web-search",
"plugins.web-search-and-fetch",
]),
@@ -278,7 +276,6 @@ describe("qa scenario catalog", () => {
const scenarioLanes = [
["openai-compatible-chat-tools", "openai-chat-tools"],
["openai-web-search-minimal", "openai-web-search-minimal"],
["openai-web-search-native-assertions", "openai-web-search-minimal"],
["openwebui-openai-compatible", "openwebui"],
["plugin-lifecycle-probe", "plugin-lifecycle-matrix"],
["packaged-bundled-plugin-install-uninstall", "bundled-plugin-install-uninstall"],

View File

@@ -5,8 +5,10 @@ scenario:
surface: model-provider
coverage:
secondary:
- plugins.web-search-and-fetch
- runtime.reasoning-and-cache-controls
- web-search.openai-native-web-search
- web-search.model-and-filter-routing
- tools.web-search
objective: Verify the OpenAI web_search minimal-reasoning E2E client distinguishes successful grounded turns from provider schema rejection.
successCriteria:
@@ -14,14 +16,21 @@ scenario:
- Reject mode fails if the agent run unexpectedly succeeds or fails for unrelated transport reasons.
- Success mode requires an `ok` agent result with the expected marker in visible reply payloads.
- Gateway ports are parsed strictly before connecting.
- A successful request must hit `/v1/responses` with native `web_search` and non-minimal reasoning.
- Large request logs are scanned without missing later success requests.
- Failure diagnostics are bounded and do not dump stale or oversized request bodies.
- Function-shaped `web_search` is rejected as native Responses proof.
docsRefs:
- docs/tools/web.md
- docs/help/testing.md
- docs/concepts/qa-e2e-automation.md
codeRefs:
- scripts/e2e/lib/openai-web-search-minimal/assertions.mjs
- scripts/e2e/lib/openai-web-search-minimal/client.mjs
- scripts/e2e/lib/openai-web-search-minimal/mock-server.mjs
- scripts/e2e/openai-web-search-minimal-docker.sh
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
- test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts
- test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts
execution:
kind: script

View File

@@ -1,34 +0,0 @@
title: OpenAI native web_search request assertions
scenario:
id: openai-web-search-native-assertions
surface: model-provider
coverage:
secondary:
- web-search.openai-native-web-search
- plugins.web-search-and-fetch
- web-search.model-and-filter-routing
- tools.web-search
objective: Verify the OpenAI web_search Docker lane assertions require native Responses web_search evidence with bounded diagnostics.
successCriteria:
- A successful request must hit `/v1/responses` with native `web_search` and non-minimal reasoning.
- Large request logs are scanned without missing later success requests.
- Failure diagnostics are bounded and do not dump stale or oversized request bodies.
- Function-shaped `web_search` is rejected as native Responses proof.
docsRefs:
- docs/tools/web.md
- docs/help/testing.md
- docs/concepts/qa-e2e-automation.md
codeRefs:
- scripts/e2e/lib/openai-web-search-minimal/assertions.mjs
- scripts/e2e/lib/openai-web-search-minimal/mock-server.mjs
- scripts/e2e/openai-web-search-minimal-docker.sh
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
- test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts
execution:
kind: script
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
summary: Runs the OpenAI web_search Docker E2E lane and asserts the emitted native Responses request log.
args:
- --lane
- openai-web-search-minimal