diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 6f678f3dfe0..83dda8ece41 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -63,8 +63,9 @@ inside the guest, runs `qa suite`, then copies the normal QA report and summary back into `.artifacts/qa-e2e/...` on the host. It reuses the same scenario-selection behavior as `qa suite` on the host. Host and Multipass suite runs execute multiple selected scenarios in parallel -with isolated gateway workers by default. Use `--concurrency ` to tune -the worker count, or `--concurrency 1` for serial execution. +with isolated gateway workers by default, up to 64 workers or the selected +scenario count. Use `--concurrency ` to tune the worker count, or +`--concurrency 1` for serial execution. Live runs forward the supported QA auth inputs that are practical for the guest: env-based provider keys, the QA live provider config path, and `CODEX_HOME` when present. Keep `--output-dir` under the repo root so the guest diff --git a/docs/help/testing.md b/docs/help/testing.md index d8c9f58ed49..072fa368f36 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -49,8 +49,9 @@ These commands sit beside the main test suites when you need QA-lab realism: - `pnpm openclaw qa suite` - Runs repo-backed QA scenarios directly on the host. - Runs multiple selected scenarios in parallel by default with isolated - gateway workers. Use `--concurrency ` to tune the worker count, or - `--concurrency 1` for the older serial lane. + gateway workers, up to 64 workers or the selected scenario count. Use + `--concurrency ` to tune the worker count, or `--concurrency 1` for + the older serial lane. - `pnpm openclaw qa suite --runner multipass` - Runs the same QA suite inside a disposable Multipass Linux VM. - Keeps the same scenario-selection behavior as `qa suite` on the host. diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index 87432150a55..1300d4c1c92 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -7,7 +7,8 @@ describe("qa suite failure reply handling", () => { const previous = process.env.OPENCLAW_QA_SUITE_CONCURRENCY; delete process.env.OPENCLAW_QA_SUITE_CONCURRENCY; try { - expect(qaSuiteTesting.normalizeQaSuiteConcurrency(undefined, 10)).toBe(4); + expect(qaSuiteTesting.normalizeQaSuiteConcurrency(undefined, 10)).toBe(10); + expect(qaSuiteTesting.normalizeQaSuiteConcurrency(undefined, 80)).toBe(64); expect(qaSuiteTesting.normalizeQaSuiteConcurrency(2.8, 10)).toBe(2); expect(qaSuiteTesting.normalizeQaSuiteConcurrency(20, 3)).toBe(3); expect(qaSuiteTesting.normalizeQaSuiteConcurrency(0, 3)).toBe(1); @@ -35,6 +36,24 @@ describe("qa suite failure reply handling", () => { expect(result).toEqual([10, 20, 30, 40]); }); + it("reads retry-after from the primary gateway error before appended logs", () => { + const error = new Error( + "rate limit exceeded for config.patch; retry after 38s\nGateway logs:\nprevious config changed since last load", + ); + + expect(qaSuiteTesting.getGatewayRetryAfterMs(error)).toBe(38_000); + expect(qaSuiteTesting.isConfigHashConflict(error)).toBe(false); + }); + + it("ignores stale retry-after text that only appears in appended gateway logs", () => { + const error = new Error( + "config changed since last load; re-run config.get and retry\nGateway logs:\nold rate limit exceeded for config.patch; retry after 38s", + ); + + expect(qaSuiteTesting.getGatewayRetryAfterMs(error)).toBe(null); + expect(qaSuiteTesting.isConfigHashConflict(error)).toBe(true); + }); + it("detects classified failure replies before a success-only outbound predicate matches", async () => { const state = createQaBusState(); state.addOutboundMessage({ diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 362ce82d72c..58cc70c1864 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -135,7 +135,7 @@ type QaRawSessionStoreEntry = { updatedAt?: number; }; -const DEFAULT_QA_SUITE_CONCURRENCY = 4; +const DEFAULT_QA_SUITE_CONCURRENCY = 64; function normalizeQaSuiteConcurrency(value: number | undefined, scenarioCount: number) { const envValue = Number(process.env.OPENCLAW_QA_SUITE_CONCURRENCY);