test: raise QA suite default concurrency

This commit is contained in:
Peter Steinberger
2026-04-10 13:45:50 +01:00
parent 4c14f55c62
commit 3027efaf21
4 changed files with 27 additions and 6 deletions

View File

@@ -63,8 +63,9 @@ inside the guest, runs `qa suite`, then copies the normal QA report and
summary back into `.artifacts/qa-e2e/...` on the host.
It reuses the same scenario-selection behavior as `qa suite` on the host.
Host and Multipass suite runs execute multiple selected scenarios in parallel
with isolated gateway workers by default. Use `--concurrency <count>` to tune
the worker count, or `--concurrency 1` for serial execution.
with isolated gateway workers by default, up to 64 workers or the selected
scenario count. Use `--concurrency <count>` to tune the worker count, or
`--concurrency 1` for serial execution.
Live runs forward the supported QA auth inputs that are practical for the
guest: env-based provider keys, the QA live provider config path, and
`CODEX_HOME` when present. Keep `--output-dir` under the repo root so the guest

View File

@@ -49,8 +49,9 @@ These commands sit beside the main test suites when you need QA-lab realism:
- `pnpm openclaw qa suite`
- Runs repo-backed QA scenarios directly on the host.
- Runs multiple selected scenarios in parallel by default with isolated
gateway workers. Use `--concurrency <count>` to tune the worker count, or
`--concurrency 1` for the older serial lane.
gateway workers, up to 64 workers or the selected scenario count. Use
`--concurrency <count>` to tune the worker count, or `--concurrency 1` for
the older serial lane.
- `pnpm openclaw qa suite --runner multipass`
- Runs the same QA suite inside a disposable Multipass Linux VM.
- Keeps the same scenario-selection behavior as `qa suite` on the host.

View File

@@ -7,7 +7,8 @@ describe("qa suite failure reply handling", () => {
const previous = process.env.OPENCLAW_QA_SUITE_CONCURRENCY;
delete process.env.OPENCLAW_QA_SUITE_CONCURRENCY;
try {
expect(qaSuiteTesting.normalizeQaSuiteConcurrency(undefined, 10)).toBe(4);
expect(qaSuiteTesting.normalizeQaSuiteConcurrency(undefined, 10)).toBe(10);
expect(qaSuiteTesting.normalizeQaSuiteConcurrency(undefined, 80)).toBe(64);
expect(qaSuiteTesting.normalizeQaSuiteConcurrency(2.8, 10)).toBe(2);
expect(qaSuiteTesting.normalizeQaSuiteConcurrency(20, 3)).toBe(3);
expect(qaSuiteTesting.normalizeQaSuiteConcurrency(0, 3)).toBe(1);
@@ -35,6 +36,24 @@ describe("qa suite failure reply handling", () => {
expect(result).toEqual([10, 20, 30, 40]);
});
it("reads retry-after from the primary gateway error before appended logs", () => {
const error = new Error(
"rate limit exceeded for config.patch; retry after 38s\nGateway logs:\nprevious config changed since last load",
);
expect(qaSuiteTesting.getGatewayRetryAfterMs(error)).toBe(38_000);
expect(qaSuiteTesting.isConfigHashConflict(error)).toBe(false);
});
it("ignores stale retry-after text that only appears in appended gateway logs", () => {
const error = new Error(
"config changed since last load; re-run config.get and retry\nGateway logs:\nold rate limit exceeded for config.patch; retry after 38s",
);
expect(qaSuiteTesting.getGatewayRetryAfterMs(error)).toBe(null);
expect(qaSuiteTesting.isConfigHashConflict(error)).toBe(true);
});
it("detects classified failure replies before a success-only outbound predicate matches", async () => {
const state = createQaBusState();
state.addOutboundMessage({

View File

@@ -135,7 +135,7 @@ type QaRawSessionStoreEntry = {
updatedAt?: number;
};
const DEFAULT_QA_SUITE_CONCURRENCY = 4;
const DEFAULT_QA_SUITE_CONCURRENCY = 64;
function normalizeQaSuiteConcurrency(value: number | undefined, scenarioCount: number) {
const envValue = Number(process.env.OPENCLAW_QA_SUITE_CONCURRENCY);