From dce871540bc96fa7317a70953ca7dd72f928d8be Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 00:54:15 -0700 Subject: [PATCH] test(qa): consolidate web search scenario (#104280) Co-authored-by: Jimmy Puckett --- .../qa-lab/src/scenario-catalog.test.ts | 7 ++-- .../runtime/openai-web-search-minimal.yaml | 9 +++++ .../openai-web-search-native-assertions.yaml | 34 ------------------- 3 files changed, 11 insertions(+), 39 deletions(-) delete mode 100644 qa/scenarios/runtime/openai-web-search-native-assertions.yaml diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index bade80f92c4a..b4b3f08a011d 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -260,11 +260,9 @@ describe("qa scenario catalog", () => { "gateway.openai-compatible-apis", "runtime.hosted-tool-use", ]); - expect(readQaScenarioById("openai-web-search-minimal").coverage?.secondary).toContain( - "runtime.reasoning-and-cache-controls", - ); - expect(readQaScenarioById("openai-web-search-native-assertions").coverage?.secondary).toEqual( + expect(readQaScenarioById("openai-web-search-minimal").coverage?.secondary).toEqual( expect.arrayContaining([ + "runtime.reasoning-and-cache-controls", "web-search.openai-native-web-search", "plugins.web-search-and-fetch", ]), @@ -278,7 +276,6 @@ describe("qa scenario catalog", () => { const scenarioLanes = [ ["openai-compatible-chat-tools", "openai-chat-tools"], ["openai-web-search-minimal", "openai-web-search-minimal"], - ["openai-web-search-native-assertions", "openai-web-search-minimal"], ["openwebui-openai-compatible", "openwebui"], ["plugin-lifecycle-probe", "plugin-lifecycle-matrix"], ["packaged-bundled-plugin-install-uninstall", "bundled-plugin-install-uninstall"], diff --git a/qa/scenarios/runtime/openai-web-search-minimal.yaml b/qa/scenarios/runtime/openai-web-search-minimal.yaml index d8ce151e20d2..9ecd21316426 100644 --- a/qa/scenarios/runtime/openai-web-search-minimal.yaml +++ b/qa/scenarios/runtime/openai-web-search-minimal.yaml @@ -5,8 +5,10 @@ scenario: surface: model-provider coverage: secondary: + - plugins.web-search-and-fetch - runtime.reasoning-and-cache-controls - web-search.openai-native-web-search + - web-search.model-and-filter-routing - tools.web-search objective: Verify the OpenAI web_search minimal-reasoning E2E client distinguishes successful grounded turns from provider schema rejection. successCriteria: @@ -14,14 +16,21 @@ scenario: - Reject mode fails if the agent run unexpectedly succeeds or fails for unrelated transport reasons. - Success mode requires an `ok` agent result with the expected marker in visible reply payloads. - Gateway ports are parsed strictly before connecting. + - A successful request must hit `/v1/responses` with native `web_search` and non-minimal reasoning. + - Large request logs are scanned without missing later success requests. + - Failure diagnostics are bounded and do not dump stale or oversized request bodies. + - Function-shaped `web_search` is rejected as native Responses proof. docsRefs: - docs/tools/web.md - docs/help/testing.md - docs/concepts/qa-e2e-automation.md codeRefs: + - scripts/e2e/lib/openai-web-search-minimal/assertions.mjs - scripts/e2e/lib/openai-web-search-minimal/client.mjs + - scripts/e2e/lib/openai-web-search-minimal/mock-server.mjs - scripts/e2e/openai-web-search-minimal-docker.sh - test/e2e/qa-lab/runtime/docker-e2e-lane.ts + - test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts - test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts execution: kind: script diff --git a/qa/scenarios/runtime/openai-web-search-native-assertions.yaml b/qa/scenarios/runtime/openai-web-search-native-assertions.yaml deleted file mode 100644 index 38691e140e88..000000000000 --- a/qa/scenarios/runtime/openai-web-search-native-assertions.yaml +++ /dev/null @@ -1,34 +0,0 @@ -title: OpenAI native web_search request assertions - -scenario: - id: openai-web-search-native-assertions - surface: model-provider - coverage: - secondary: - - web-search.openai-native-web-search - - plugins.web-search-and-fetch - - web-search.model-and-filter-routing - - tools.web-search - objective: Verify the OpenAI web_search Docker lane assertions require native Responses web_search evidence with bounded diagnostics. - successCriteria: - - A successful request must hit `/v1/responses` with native `web_search` and non-minimal reasoning. - - Large request logs are scanned without missing later success requests. - - Failure diagnostics are bounded and do not dump stale or oversized request bodies. - - Function-shaped `web_search` is rejected as native Responses proof. - docsRefs: - - docs/tools/web.md - - docs/help/testing.md - - docs/concepts/qa-e2e-automation.md - codeRefs: - - scripts/e2e/lib/openai-web-search-minimal/assertions.mjs - - scripts/e2e/lib/openai-web-search-minimal/mock-server.mjs - - scripts/e2e/openai-web-search-minimal-docker.sh - - test/e2e/qa-lab/runtime/docker-e2e-lane.ts - - test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts - execution: - kind: script - path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts - summary: Runs the OpenAI web_search Docker E2E lane and asserts the emitted native Responses request log. - args: - - --lane - - openai-web-search-minimal