From 66b4dcf184be8ccc225989901df7e974891c81bc Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 6 Jul 2026 13:05:34 -0700 Subject: [PATCH] chore(docker): execute Compose and package artifact proofs (#101045) * test(docker): prove Compose and package artifacts * test(docker): use bounded container cleanup * test(docker): bound package proof launch * test(qa): keep Docker artifact proofs release-only * test(docker): simplify artifact proof names * test(docker): satisfy identity writer lint --- extensions/qa-lab/src/coverage-report.test.ts | 19 +- package.json | 2 + qa/scenarios/runtime/compose-setup.yaml | 35 +++ .../runtime/docker-compose-setup.yaml | 28 --- .../runtime/docker-package-install.yaml | 36 +++ .../runtime/package-openclaw-for-docker.yaml | 27 -- scripts/e2e/compose-setup.sh | 89 +++++++ scripts/e2e/docker-package-install.sh | 68 +++++ .../docker-artifact-proof/write-identities.ts | 157 ++++++++++++ scripts/lib/docker-e2e-scenarios.mjs | 18 ++ .../runtime/docker-artifact-proof.test.ts | 63 +++++ .../qa-lab/runtime/docker-artifact-proof.ts | 236 ++++++++++++++++++ test/scripts/docker-build-helper.test.ts | 22 ++ test/scripts/docker-e2e-plan.test.ts | 36 +++ 14 files changed, 780 insertions(+), 56 deletions(-) create mode 100644 qa/scenarios/runtime/compose-setup.yaml delete mode 100644 qa/scenarios/runtime/docker-compose-setup.yaml create mode 100644 qa/scenarios/runtime/docker-package-install.yaml delete mode 100644 qa/scenarios/runtime/package-openclaw-for-docker.yaml create mode 100755 scripts/e2e/compose-setup.sh create mode 100755 scripts/e2e/docker-package-install.sh create mode 100644 scripts/e2e/lib/docker-artifact-proof/write-identities.ts create mode 100644 test/e2e/qa-lab/runtime/docker-artifact-proof.test.ts create mode 100644 test/e2e/qa-lab/runtime/docker-artifact-proof.ts diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index 7faac93ad075..63d20bfa9f83 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -143,7 +143,8 @@ function scenarioWithCoverage(params: { describe("qa coverage report", () => { it("groups scenario coverage metadata by theme and surface", () => { - const inventory = buildQaCoverageInventory(readQaScenarioPack().scenarios); + const scenarios = readQaScenarioPack().scenarios; + const inventory = buildQaCoverageInventory(scenarios); expect(inventory.scenarioCount).toBeGreaterThan(0); expect(inventory.coverageIdCount).toBeGreaterThan(0); @@ -169,6 +170,22 @@ describe("qa coverage report", () => { ).toMatchObject({ channelDriver: "live", }); + for (const [categoryId, scenarioRef] of [ + ["docker-podman-hosting.container-setup", "qa/scenarios/runtime/compose-setup.yaml"], + [ + "docker-podman-hosting.image-release-and-validation", + "qa/scenarios/runtime/docker-package-install.yaml", + ], + ] as const) { + const category = inventory.scorecardTaxonomy.categories.find( + (entry) => entry.id === categoryId, + ); + expect(category?.profiles).toContain("release"); + expect(category?.profiles).not.toContain("smoke-ci"); + expect(scenarios.find((scenario) => scenario.sourcePath === scenarioRef)?.category).toBe( + categoryId, + ); + } expect( inventory.scorecardTaxonomy.profiles.find((profile) => profile.id === "all"), ).toMatchObject({ diff --git a/package.json b/package.json index 47cfa490fcff..5e0975d3f6b6 100644 --- a/package.json +++ b/package.json @@ -1796,6 +1796,7 @@ "test:docker:cron-mcp-cleanup": "bash scripts/e2e/cron-mcp-cleanup-docker.sh", "test:docker:codex-media-path": "bash scripts/e2e/codex-media-path-docker.sh", "test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh", + "test:docker:compose-setup": "bash scripts/e2e/compose-setup.sh", "test:docker:e2e-build": "bash scripts/e2e/build-image.sh", "test:docker:gateway-network": "bash scripts/e2e/gateway-network-docker.sh", "test:docker:kitchen-sink-plugin": "bash scripts/e2e/kitchen-sink-plugin-docker.sh", @@ -1848,6 +1849,7 @@ "test:docker:openai-image-auth": "bash scripts/e2e/openai-image-auth-docker.sh", "test:docker:openai-web-search-minimal": "bash scripts/e2e/openai-web-search-minimal-docker.sh", "test:docker:openwebui": "bash scripts/e2e/openwebui-docker.sh", + "test:docker:package-install": "bash scripts/e2e/docker-package-install.sh", "test:docker:plugin-binding-command-escape": "bash scripts/e2e/plugin-binding-command-escape-docker.sh", "test:docker:plugin-lifecycle-matrix": "bash scripts/e2e/plugin-lifecycle-matrix-docker.sh", "test:docker:plugin-update": "bash scripts/e2e/plugin-update-unchanged-docker.sh", diff --git a/qa/scenarios/runtime/compose-setup.yaml b/qa/scenarios/runtime/compose-setup.yaml new file mode 100644 index 000000000000..f2d658f6afee --- /dev/null +++ b/qa/scenarios/runtime/compose-setup.yaml @@ -0,0 +1,35 @@ +title: Docker Compose setup evidence + +scenario: + id: compose-setup + surface: docker-podman-hosting + category: docker-podman-hosting.container-setup + coverage: + secondary: + - docker.compose + objective: Launch the documented Docker Compose topology from the package-backed functional image and record the exact package, image, and service container identities. + successCriteria: + - The scheduler builds the real package artifact and package-backed functional image through the canonical Docker planner. + - "`docker compose up` starts the documented `openclaw-gateway` service with isolated state, config, workspace, and auth-profile mounts." + - The Gateway reaches healthy state and the documented container-side health command passes. + - "The `openclaw-cli` Compose service runs successfully in the Gateway network namespace." + - Evidence records the package digest and version, functional image ID, and Gateway and CLI container IDs. + docsRefs: + - docs/install/docker.md + - docs/help/testing.md + codeRefs: + - docker-compose.yml + - scripts/e2e/compose-setup.sh + - scripts/lib/docker-e2e-scenarios.mjs + - scripts/package-openclaw-for-docker.mjs + - test/e2e/qa-lab/runtime/docker-artifact-proof.ts + execution: + kind: script + path: test/e2e/qa-lab/runtime/docker-artifact-proof.ts + summary: Builds the package-backed functional image, launches the documented Compose services, validates health, and records artifact identities. + timeoutMs: 1800000 + args: + - --artifact-base + - ${outputDir} + - --lane + - compose-setup diff --git a/qa/scenarios/runtime/docker-compose-setup.yaml b/qa/scenarios/runtime/docker-compose-setup.yaml deleted file mode 100644 index 0075144245b4..000000000000 --- a/qa/scenarios/runtime/docker-compose-setup.yaml +++ /dev/null @@ -1,28 +0,0 @@ -title: Docker setup and Compose evidence - -scenario: - id: docker-compose-setup - surface: docker-podman-hosting - category: docker-podman-hosting.container-setup - coverage: - secondary: - - docker.compose - - docker.local-image-setup-script - - docker.only-first-run-notes - objective: Link Docker Compose setup e2e coverage to Docker maturity accounting. - successCriteria: - - The Docker setup script builds local images, reuses preloaded GHCR images in offline mode, and preserves BuildKit behavior. - - First-run setup emits Docker-only post-start notes without printing the gateway token. - - Docker Compose gateway and CLI service command shape stays in sync. - - Compose service env, token, auth-profile, timezone, and optional env-file defaults stay aligned. - - Container-side state, config, and workspace paths override host `.env` values. - docsRefs: - - docs/install/docker.md - - docs/help/testing.md - codeRefs: - - scripts/docker/setup.sh - - src/docker-setup.e2e.test.ts - execution: - kind: vitest - path: src/docker-setup.e2e.test.ts - summary: Vitest e2e coverage for Docker setup script, first-run notes, Compose service shape, and mount/env contracts. diff --git a/qa/scenarios/runtime/docker-package-install.yaml b/qa/scenarios/runtime/docker-package-install.yaml new file mode 100644 index 000000000000..71575872e7b9 --- /dev/null +++ b/qa/scenarios/runtime/docker-package-install.yaml @@ -0,0 +1,36 @@ +title: Docker package install evidence + +scenario: + id: docker-package-install + surface: docker-podman-hosting + category: docker-podman-hosting.image-release-and-validation + coverage: + secondary: + - docker.package-artifact-generation + - cli.package-manager-installs + objective: Build the real OpenClaw package artifact, install and execute it in the scheduler-selected bare container image, and record exact package, image, and container identities. + successCriteria: + - The canonical Docker package helper builds and validates the real npm tarball. + - The scheduler prepares the package and bare target image through the shared planner and catalog. + - A clean target container installs the tarball with npm and executes the installed OpenClaw CLI. + - The installed package name and version match the tarball metadata. + - Evidence records the package digest and version, bare image ID, target container ID, installed package root, and executed CLI version. + docsRefs: + - docs/install/updating.md + - docs/help/testing.md + - docs/concepts/qa-e2e-automation.md + codeRefs: + - scripts/package-openclaw-for-docker.mjs + - scripts/e2e/docker-package-install.sh + - scripts/lib/docker-e2e-scenarios.mjs + - test/e2e/qa-lab/runtime/docker-artifact-proof.ts + execution: + kind: script + path: test/e2e/qa-lab/runtime/docker-artifact-proof.ts + summary: Builds the real package artifact, installs and executes it in a clean target container, and records artifact identities. + timeoutMs: 1800000 + args: + - --artifact-base + - ${outputDir} + - --lane + - docker-package-install diff --git a/qa/scenarios/runtime/package-openclaw-for-docker.yaml b/qa/scenarios/runtime/package-openclaw-for-docker.yaml deleted file mode 100644 index fecf7dfc71d8..000000000000 --- a/qa/scenarios/runtime/package-openclaw-for-docker.yaml +++ /dev/null @@ -1,27 +0,0 @@ -title: Docker package artifact QA evidence - -scenario: - id: package-openclaw-for-docker - surface: docker-podman-hosting - coverage: - secondary: - - docker.package-artifact-generation - - cli.package-manager-installs - - runtime.package-update - objective: Exercise bounded OpenClaw package artifact generation through QA Lab evidence. - successCriteria: - - Package artifact output flags are parsed strictly. - - The Docker package path uses the single bounded build-all step before npm pack. - - Changelog trimming is restored after successful and failed ignore-scripts packaging. - - Timed-out and externally terminated child process groups are cleaned up without leaked descendants. - - Captured command output is bounded. - docsRefs: - - docs/install/updating.md - - docs/help/testing.md - - docs/concepts/qa-e2e-automation.md - codeRefs: - - test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts - execution: - kind: vitest - path: test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts - summary: Vitest coverage for Docker package artifact creation and cleanup behavior. diff --git a/scripts/e2e/compose-setup.sh b/scripts/e2e/compose-setup.sh new file mode 100755 index 000000000000..04ef24e25648 --- /dev/null +++ b/scripts/e2e/compose-setup.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh" + +IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-docker-e2e-functional:local")" +PACKAGE_TGZ="$(docker_e2e_prepare_package_tgz compose-setup "${OPENCLAW_CURRENT_PACKAGE_TGZ:-}")" +IDENTITY_PATH="${OPENCLAW_DOCKER_ARTIFACT_IDENTITY_PATH:-$ROOT_DIR/.artifacts/docker-tests/compose-setup-identities.json}" +PROJECT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-compose-proof.XXXXXX")" +PROJECT_NAME="openclaw-compose-proof-$$" +CLI_NAME="$PROJECT_NAME-cli-proof" +TOKEN="compose-proof-$$-$(date +%s)" +COMPOSE=(docker compose --project-name "$PROJECT_NAME" --project-directory "$PROJECT_DIR" -f "$ROOT_DIR/docker-compose.yml") + +cleanup() { + docker_e2e_docker_cmd rm -f "$CLI_NAME" >/dev/null 2>&1 || true + "${COMPOSE[@]}" down --remove-orphans --volumes >/dev/null 2>&1 || true + docker_e2e_cleanup_package_tgz "$PACKAGE_TGZ" + rm -rf "$PROJECT_DIR" +} +trap cleanup EXIT + +mkdir -p "$PROJECT_DIR/config/workspace" "$PROJECT_DIR/auth-profile" +chmod -R 0777 "$PROJECT_DIR/config" "$PROJECT_DIR/auth-profile" +cat >"$PROJECT_DIR/config/openclaw.json" <&2 + exit 1 +fi + +for _ in $(seq 1 180); do + health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$GATEWAY_ID")" + if [ "$health" = "healthy" ]; then + break + fi + if [ "$health" = "unhealthy" ] || [ "$health" = "exited" ] || [ "$health" = "dead" ]; then + "${COMPOSE[@]}" logs --no-color openclaw-gateway >&2 + exit 1 + fi + sleep 1 +done +if [ "$(docker inspect --format '{{.State.Health.Status}}' "$GATEWAY_ID")" != "healthy" ]; then + "${COMPOSE[@]}" logs --no-color openclaw-gateway >&2 + echo "Compose gateway did not become healthy" >&2 + exit 1 +fi + +"${COMPOSE[@]}" exec -T openclaw-gateway node dist/index.js health --token "$TOKEN" +"${COMPOSE[@]}" run -T --no-deps --name "$CLI_NAME" openclaw-cli health --token "$TOKEN" +GATEWAY_VERSION="$("${COMPOSE[@]}" exec -T openclaw-gateway node -p "require('./package.json').version")" + +node --import tsx "$ROOT_DIR/scripts/e2e/lib/docker-artifact-proof/write-identities.ts" \ + --scenario compose-setup \ + --output "$IDENTITY_PATH" \ + --image "$IMAGE_NAME" \ + --package "$PACKAGE_TGZ" \ + --container "gateway=$GATEWAY_ID" \ + --container "cli=$CLI_NAME" \ + --detail "gateway:openclawVersion=$GATEWAY_VERSION" \ + --detail "gateway:health=healthy" \ + --detail "cli:healthCommand=passed" + +echo "Docker Compose setup proof passed." diff --git a/scripts/e2e/docker-package-install.sh b/scripts/e2e/docker-package-install.sh new file mode 100755 index 000000000000..16fe9c025eb7 --- /dev/null +++ b/scripts/e2e/docker-package-install.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh" + +IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-docker-e2e-bare:local")" +PACKAGE_TGZ="$(docker_e2e_prepare_package_tgz docker-package-install "${OPENCLAW_CURRENT_PACKAGE_TGZ:-}")" +IDENTITY_PATH="${OPENCLAW_DOCKER_ARTIFACT_IDENTITY_PATH:-$ROOT_DIR/.artifacts/docker-tests/docker-package-install-identities.json}" +CONTAINER_NAME="openclaw-package-proof-$$" +DOCKER_RUN_TIMEOUT="${OPENCLAW_DOCKER_PACKAGE_INSTALL_RUN_TIMEOUT:-120s}" + +cleanup() { + docker_e2e_docker_cmd rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + docker_e2e_cleanup_package_tgz "$PACKAGE_TGZ" +} +trap cleanup EXIT + +docker_e2e_build_or_reuse "$IMAGE_NAME" docker-package-install "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" bare + +echo "Installing the real OpenClaw package artifact in the target container..." +DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run -d \ + --name "$CONTAINER_NAME" \ + -v "$PACKAGE_TGZ:/tmp/openclaw-current.tgz:ro" \ + "$IMAGE_NAME" \ + bash -lc ' + set -euo pipefail + npm install -g --prefix /tmp/openclaw-proof /tmp/openclaw-current.tgz --no-fund --no-audit + package_root=/tmp/openclaw-proof/lib/node_modules/openclaw + "$package_root/openclaw.mjs" --version > /tmp/openclaw-version + "$package_root/openclaw.mjs" --help > /tmp/openclaw-help + test -s /tmp/openclaw-help + touch /tmp/openclaw-proof-ready + exec sleep infinity + ' >/dev/null + +for _ in $(seq 1 240); do + if docker exec "$CONTAINER_NAME" test -f /tmp/openclaw-proof-ready; then + break + fi + if [ "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_NAME")" != "true" ]; then + docker logs "$CONTAINER_NAME" >&2 + exit 1 + fi + sleep 1 +done +docker exec "$CONTAINER_NAME" test -f /tmp/openclaw-proof-ready + +INSTALLED_VERSION="$(docker exec "$CONTAINER_NAME" cat /tmp/openclaw-version | tr -d '\r\n')" +PACKAGE_ROOT="/tmp/openclaw-proof/lib/node_modules/openclaw" +PACKAGE_VERSION="$(docker exec "$CONTAINER_NAME" node -p "require('$PACKAGE_ROOT/package.json').version")" +if [[ "$INSTALLED_VERSION" != *"$PACKAGE_VERSION"* ]]; then + echo "installed CLI output $INSTALLED_VERSION does not contain package version $PACKAGE_VERSION" >&2 + exit 1 +fi + +node --import tsx "$ROOT_DIR/scripts/e2e/lib/docker-artifact-proof/write-identities.ts" \ + --scenario docker-package-install \ + --output "$IDENTITY_PATH" \ + --image "$IMAGE_NAME" \ + --package "$PACKAGE_TGZ" \ + --container "target=$CONTAINER_NAME" \ + --detail "target:installedPackageRoot=$PACKAGE_ROOT" \ + --detail "target:installedPackageVersion=$PACKAGE_VERSION" \ + --detail "target:openclawVersion=$INSTALLED_VERSION" \ + --detail "target:helpCommand=passed" + +echo "Package artifact container proof passed." diff --git a/scripts/e2e/lib/docker-artifact-proof/write-identities.ts b/scripts/e2e/lib/docker-artifact-proof/write-identities.ts new file mode 100644 index 000000000000..dc1c9db3a9c1 --- /dev/null +++ b/scripts/e2e/lib/docker-artifact-proof/write-identities.ts @@ -0,0 +1,157 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +type ContainerOption = { + name: string; + role: string; +}; + +type WriterOptions = { + containers: ContainerOption[]; + details: Map>; + image?: string; + output?: string; + package?: string; + scenario?: string; +}; + +type RequiredWriterOptions = WriterOptions & { + image: string; + output: string; + package: string; + scenario: string; +}; + +type DockerInspect = { + Id: string; + Image?: string; + Name?: string; + RepoDigests?: string[]; + State?: { Status?: string }; +}; + +type PackageManifest = { + name?: unknown; + version?: unknown; +}; + +function readValue(args: string[], index: number, option: string): string { + const value = args[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`${option} requires a value`); + } + return value; +} + +function parseArgs(args: string[]): RequiredWriterOptions { + const options: WriterOptions = { containers: [], details: new Map() }; + for (let index = 0; index < args.length; index += 1) { + const option = args[index]; + if (!option) { + continue; + } + const value = readValue(args, index, option); + index += 1; + if (option === "--scenario") { + options.scenario = value; + } else if (option === "--output") { + options.output = value; + } else if (option === "--image") { + options.image = value; + } else if (option === "--package") { + options.package = value; + } else if (option === "--container") { + const separator = value.indexOf("="); + if (separator < 1 || separator === value.length - 1) { + throw new Error("--container must use role=name"); + } + options.containers.push({ + role: value.slice(0, separator), + name: value.slice(separator + 1), + }); + } else if (option === "--detail") { + const roleSeparator = value.indexOf(":"); + const valueSeparator = value.indexOf("=", roleSeparator + 1); + if (roleSeparator < 1 || valueSeparator < roleSeparator + 2) { + throw new Error("--detail must use role:key=value"); + } + const role = value.slice(0, roleSeparator); + const key = value.slice(roleSeparator + 1, valueSeparator); + const detailValue = value.slice(valueSeparator + 1); + const roleDetails = options.details.get(role) ?? {}; + roleDetails[key] = detailValue; + options.details.set(role, roleDetails); + } else { + throw new Error(`unknown option: ${option}`); + } + } + for (const required of ["scenario", "output", "image", "package"] as const) { + if (!options[required]) { + throw new Error(`--${required} is required`); + } + } + if (options.containers.length === 0) { + throw new Error("at least one --container is required"); + } + return options as RequiredWriterOptions; +} + +function run(command: string, args: string[]): string { + return execFileSync(command, args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).trim(); +} + +function inspectDockerObject(reference: string): DockerInspect { + const [result] = JSON.parse(run("docker", ["inspect", reference])) as DockerInspect[]; + if (!result) { + throw new Error(`docker inspect returned no result for ${reference}`); + } + return result; +} + +function readPackageIdentity(packagePath: string) { + const packageJson = JSON.parse( + run("tar", ["-xOf", packagePath, "package/package.json"]), + ) as PackageManifest; + if (typeof packageJson.name !== "string" || typeof packageJson.version !== "string") { + throw new Error("package artifact manifest is missing name or version"); + } + const bytes = fs.readFileSync(packagePath); + return { + fileName: path.basename(packagePath), + name: packageJson.name, + sha256: createHash("sha256").update(bytes).digest("hex"), + sizeBytes: bytes.byteLength, + version: packageJson.version, + }; +} + +const options = parseArgs(process.argv.slice(2)); +const image = inspectDockerObject(options.image); +const containers = options.containers.map(({ name, role }) => { + const container = inspectDockerObject(name); + return { + details: options.details.get(role) ?? {}, + id: container.Id, + imageId: container.Image, + name: (container.Name ?? name).replace(/^\//u, ""), + role, + state: container.State?.Status, + }; +}); +const identity = { + containers, + image: { + id: image.Id, + reference: options.image, + repoDigests: image.RepoDigests ?? [], + }, + package: readPackageIdentity(options.package), + scenarioId: options.scenario, +}; +fs.mkdirSync(path.dirname(options.output), { recursive: true }); +fs.writeFileSync(options.output, `${JSON.stringify(identity, null, 2)}\n`); +console.log( + `artifact identities: package=${identity.package.name}@${identity.package.version} sha256=${identity.package.sha256} image=${identity.image.id} containers=${containers.map((container) => `${container.role}:${container.id}`).join(",")}`, +); diff --git a/scripts/lib/docker-e2e-scenarios.mjs b/scripts/lib/docker-e2e-scenarios.mjs index c2ce5f216dde..7b5b9bfe4905 100644 --- a/scripts/lib/docker-e2e-scenarios.mjs +++ b/scripts/lib/docker-e2e-scenarios.mjs @@ -262,6 +262,24 @@ function kitchenSinkRpcLane() { } export const mainLanes = [ + serviceLane( + "compose-setup", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:compose-setup", + { + stateScenario: "empty", + timeoutMs: 20 * 60 * 1000, + weight: 3, + }, + ), + npmLane( + "docker-package-install", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:package-install", + { + stateScenario: "empty", + timeoutMs: 20 * 60 * 1000, + weight: 3, + }, + ), liveLane("live-models", liveDockerScriptCommand("test-live-models-docker.sh"), { providers: ["claude-cli", "google-gemini-cli"], timeoutMs: LIVE_PROFILE_TIMEOUT_MS, diff --git a/test/e2e/qa-lab/runtime/docker-artifact-proof.test.ts b/test/e2e/qa-lab/runtime/docker-artifact-proof.test.ts new file mode 100644 index 000000000000..bba01b4e56af --- /dev/null +++ b/test/e2e/qa-lab/runtime/docker-artifact-proof.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { + formatDockerArtifactIdentityDetails, + parseDockerArtifactProofOptions, +} from "./docker-artifact-proof.js"; + +describe("Docker artifact proof producer", () => { + it("parses the two canonical artifact lanes", () => { + expect( + parseDockerArtifactProofOptions([ + "--artifact-base", + ".artifacts/proof", + "--lane", + "compose-setup", + ]).lane, + ).toBe("compose-setup"); + expect( + parseDockerArtifactProofOptions([ + "--artifact-base", + ".artifacts/proof", + "--lane", + "docker-package-install", + ]).lane, + ).toBe("docker-package-install"); + }); + + it("rejects non-artifact Docker lanes", () => { + expect(() => + parseDockerArtifactProofOptions([ + "--artifact-base", + ".artifacts/proof", + "--lane", + "gateway-network", + ]), + ).toThrow("unsupported Docker artifact proof lane"); + }); + + it("formats package, image, and container identities", () => { + expect( + formatDockerArtifactIdentityDetails({ + containers: [ + { + details: { health: "healthy" }, + id: "container1234567890", + imageId: "sha256:image", + name: "gateway", + role: "gateway", + state: "running", + }, + ], + image: { id: "sha256:image", reference: "openclaw:functional", repoDigests: [] }, + package: { + fileName: "openclaw-current.tgz", + name: "openclaw", + sha256: "a".repeat(64), + sizeBytes: 42, + version: "2026.7.6", + }, + scenarioId: "compose-setup", + }), + ).toContain("containers=gateway=container123"); + }); +}); diff --git a/test/e2e/qa-lab/runtime/docker-artifact-proof.ts b/test/e2e/qa-lab/runtime/docker-artifact-proof.ts new file mode 100644 index 000000000000..fa32ce70fb41 --- /dev/null +++ b/test/e2e/qa-lab/runtime/docker-artifact-proof.ts @@ -0,0 +1,236 @@ +// Runs package-backed Docker artifact lanes and writes bounded QA evidence. +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + QA_EVIDENCE_FILENAME, + type QaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/api.js"; +import { createQaScriptEvidenceWriter } from "./script-evidence.js"; + +const SOURCE_PATH = "test/e2e/qa-lab/runtime/docker-artifact-proof.ts"; + +const PROOFS = { + "compose-setup": { + secondaryCoverageIds: ["docker.compose"], + title: "Docker Compose setup evidence", + }, + "docker-package-install": { + secondaryCoverageIds: ["docker.package-artifact-generation", "cli.package-manager-installs"], + title: "Docker package install evidence", + }, +} as const; + +type DockerArtifactProofLane = keyof typeof PROOFS; + +type ProducerOptions = { + artifactBase: string; + lane: DockerArtifactProofLane; + repoRoot: string; +}; + +type ArtifactIdentity = { + containers: Array<{ + details: Record; + id: string; + imageId: string; + name: string; + role: string; + state: string; + }>; + image: { id: string; reference: string; repoDigests: string[] }; + package: { fileName: string; name: string; sha256: string; sizeBytes: number; version: string }; + scenarioId: string; +}; + +function formatErrorMessage(error: unknown) { + return error instanceof Error ? error.message : String(error); +} + +function isProofLane(value: string): value is DockerArtifactProofLane { + return Object.hasOwn(PROOFS, value); +} + +export function parseDockerArtifactProofOptions(args: string[]): ProducerOptions { + let artifactBase: string | undefined; + let lane: DockerArtifactProofLane | undefined; + for (let index = 0; index < args.length; index += 1) { + const option = args[index]; + const value = args[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`${option} requires a value`); + } + index += 1; + if (option === "--artifact-base") { + artifactBase = value; + } else if (option === "--lane") { + if (!isProofLane(value)) { + throw new Error(`unsupported Docker artifact proof lane: ${value}`); + } + lane = value; + } else { + throw new Error(`unknown argument: ${option}`); + } + } + if (!artifactBase) { + throw new Error("--artifact-base is required"); + } + if (!lane) { + throw new Error("--lane is required"); + } + return { artifactBase: path.resolve(artifactBase), lane, repoRoot: process.cwd() }; +} + +function assertIdentity(identity: ArtifactIdentity, lane: DockerArtifactProofLane) { + if (identity.scenarioId !== lane) { + throw new Error(`identity scenario ${identity.scenarioId} does not match ${lane}`); + } + if ( + !identity.package.name || + !identity.package.version || + !/^[a-f0-9]{64}$/u.test(identity.package.sha256) + ) { + throw new Error("identity package metadata is incomplete"); + } + if (!identity.image.id || identity.containers.length === 0) { + throw new Error("identity image or container metadata is incomplete"); + } +} + +export function formatDockerArtifactIdentityDetails(identity: ArtifactIdentity) { + const containers = identity.containers + .map((container) => `${container.role}=${container.id.slice(0, 12)}`) + .join(", "); + return [ + `package=${identity.package.name}@${identity.package.version}`, + `sha256=${identity.package.sha256}`, + `image=${identity.image.reference}@${identity.image.id}`, + `containers=${containers}`, + ].join("; "); +} + +async function runScheduler(options: ProducerOptions, appendLog: (chunk: unknown) => void) { + const dockerRunDir = path.join(options.artifactBase, "docker-run"); + const identityPath = path.join(options.artifactBase, "artifact-identities.json"); + await fs.mkdir(dockerRunDir, { recursive: true }); + let packageTgz = process.env.OPENCLAW_CURRENT_PACKAGE_TGZ; + if (packageTgz) { + const packageDir = path.join(dockerRunDir, "openclaw-package"); + const evidencePackageTgz = path.join(packageDir, "openclaw-current.tgz"); + const sourcePackageTgz = path.resolve(packageTgz); + await fs.mkdir(packageDir, { recursive: true }); + if (sourcePackageTgz !== evidencePackageTgz) { + await fs.copyFile(sourcePackageTgz, evidencePackageTgz); + } + packageTgz = evidencePackageTgz; + } + return await new Promise<{ + code: number | null; + identityPath: string; + signal: NodeJS.Signals | null; + }>((resolve, reject) => { + const child = spawn(process.execPath, ["scripts/test-docker-all.mjs"], { + cwd: options.repoRoot, + env: { + ...process.env, + OPENCLAW_DOCKER_ALL_BUILD: "1", + OPENCLAW_DOCKER_ALL_DRY_RUN: "0", + OPENCLAW_DOCKER_ALL_LANES: options.lane, + OPENCLAW_DOCKER_ALL_LOG_DIR: dockerRunDir, + OPENCLAW_DOCKER_ALL_PARALLELISM: "1", + OPENCLAW_DOCKER_ALL_PREFLIGHT: "1", + OPENCLAW_DOCKER_ALL_TIMINGS_FILE: path.join(dockerRunDir, "lane-timings.json"), + OPENCLAW_DOCKER_ARTIFACT_IDENTITY_PATH: identityPath, + ...(packageTgz ? { OPENCLAW_CURRENT_PACKAGE_TGZ: packageTgz } : {}), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.on("error", reject); + child.stdout.on("data", (chunk: Buffer) => { + process.stdout.write(chunk); + appendLog(chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + process.stderr.write(chunk); + appendLog(chunk); + }); + child.on("exit", (code, signal) => resolve({ code, identityPath, signal })); + }); +} + +export async function runDockerArtifactProofProducer( + options: ProducerOptions, +): Promise { + const proof = PROOFS[options.lane]; + const writer = createQaScriptEvidenceWriter({ + artifactBase: options.artifactBase, + logFileName: "docker-artifact-proof.log", + primaryModel: "docker/package-artifact", + providerMode: "mock-openai", + repoRoot: options.repoRoot, + target: { + codeRefs: [ + SOURCE_PATH, + "scripts/test-docker-all.mjs", + "scripts/lib/docker-e2e-plan.mjs", + "scripts/lib/docker-e2e-scenarios.mjs", + "scripts/package-openclaw-for-docker.mjs", + ], + docsRefs: ["docs/install/docker.md", "docs/help/testing.md"], + id: options.lane, + secondaryCoverageIds: proof.secondaryCoverageIds, + sourcePath: SOURCE_PATH, + title: proof.title, + }, + }); + const startedAt = Date.now(); + try { + const result = await runScheduler(options, (chunk) => writer.appendLog(chunk)); + if (result.code !== 0 || result.signal) { + throw new Error( + `Docker scheduler lane ${options.lane} failed: code=${String(result.code)} signal=${String(result.signal)}`, + ); + } + const identity = JSON.parse(await fs.readFile(result.identityPath, "utf8")) as ArtifactIdentity; + assertIdentity(identity, options.lane); + const packagePath = path.join("docker-run", "openclaw-package", "openclaw-current.tgz"); + return await writer.write({ + artifacts: [ + { kind: "identity", filePath: "artifact-identities.json" }, + { kind: "package", filePath: packagePath }, + { kind: "summary", filePath: path.join("docker-run", "summary.json") }, + ], + details: formatDockerArtifactIdentityDetails(identity), + durationMs: Math.max(1, Date.now() - startedAt), + status: "pass", + }); + } catch (error) { + const details = formatErrorMessage(error); + writer.appendLog(`\nfail: ${details}\n`); + return await writer.write({ + details, + durationMs: Math.max(1, Date.now() - startedAt), + status: "fail", + }); + } +} + +async function main(argv: string[]) { + const evidence = await runDockerArtifactProofProducer(parseDockerArtifactProofOptions(argv)); + const status = evidence.entries[0]?.result.status; + console.log(`Docker artifact proof evidence: ${QA_EVIDENCE_FILENAME}`); + console.log(`Docker artifact proof status: ${status}`); + return status === "pass" ? 0 : 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)) + .then((exitCode) => { + process.exitCode = exitCode; + }) + .catch((error: unknown) => { + console.error(formatErrorMessage(error)); + process.exitCode = 1; + }); +} diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 479aeaf888bf..40e31b5de101 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -20,6 +20,8 @@ const DOCKER_ALL_SCHEDULER_PATH = "scripts/test-docker-all.mjs"; const DOCKER_E2E_PACKAGE_HELPER_PATH = "scripts/lib/docker-e2e-package.sh"; const DOCKER_E2E_IMAGE_HELPER_PATH = "scripts/lib/docker-e2e-image.sh"; const DOCKER_E2E_SCENARIOS_PATH = "scripts/lib/docker-e2e-scenarios.mjs"; +const COMPOSE_SETUP_E2E_PATH = "scripts/e2e/compose-setup.sh"; +const DOCKER_PACKAGE_INSTALL_E2E_PATH = "scripts/e2e/docker-package-install.sh"; const INSTALL_E2E_RUNNER_PATH = "scripts/docker/install-sh-e2e/run.sh"; const CLEANUP_DOCKER_SMOKE_PATH = "scripts/test-cleanup-docker.sh"; const INSTALL_E2E_DOCKER_SMOKE_PATH = "scripts/test-install-sh-e2e-docker.sh"; @@ -3769,6 +3771,26 @@ output="$(cat "$sampler_log")" expect(runner, path).not.toMatch(/(^|\n)\s*docker rm -f "\$CONTAINER_NAME"/u); expect(runner, path).toContain('docker_e2e_docker_cmd rm -f "$CONTAINER_NAME"'); } + + const composeRunner = readFileSync(COMPOSE_SETUP_E2E_PATH, "utf8"); + expect(composeRunner).not.toMatch(/(^|\n)\s*docker rm -f "\$CLI_NAME"/u); + expect(composeRunner).toContain('docker_e2e_docker_cmd rm -f "$CLI_NAME"'); + + const packageRunner = readFileSync(DOCKER_PACKAGE_INSTALL_E2E_PATH, "utf8"); + expect(packageRunner).not.toMatch(/(^|\n)\s*docker rm -f "\$CONTAINER_NAME"/u); + expect(packageRunner).toContain('docker_e2e_docker_cmd rm -f "$CONTAINER_NAME"'); + expect(packageRunner).toContain( + 'DOCKER_RUN_TIMEOUT="${OPENCLAW_DOCKER_PACKAGE_INSTALL_RUN_TIMEOUT:-120s}"', + ); + expect(packageRunner).toContain( + 'DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run -d', + ); + expect(packageRunner).not.toMatch(/(^|\n)docker run -d/u); + for (const runner of [composeRunner, packageRunner]) { + expect(runner).toContain( + 'node --import tsx "$ROOT_DIR/scripts/e2e/lib/docker-artifact-proof/write-identities.ts"', + ); + } }); it("routes the gateway network client through the timeout-aware run helper", () => { diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index 5e1c86b63993..8d725fdead3e 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -97,6 +97,42 @@ function bundledPluginSweepLane(index: number): ReturnType } describe("scripts/lib/docker-e2e-plan", () => { + it("plans package-backed Compose and package artifact proofs", () => { + const plan = planFor({ + selectedLaneNames: ["compose-setup", "docker-package-install"], + }); + + expect(plan.lanes.map(summarizeLane)).toEqual([ + { + command: "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:compose-setup", + imageKind: "functional", + live: false, + name: "compose-setup", + resources: ["docker", "service"], + stateScenario: "empty", + timeoutMs: 1_200_000, + weight: 3, + }, + { + command: "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:package-install", + imageKind: "bare", + live: false, + name: "docker-package-install", + resources: ["docker", "npm"], + stateScenario: "empty", + timeoutMs: 1_200_000, + weight: 3, + }, + ]); + expect(plan.needs).toEqual({ + bareImage: true, + e2eImage: true, + functionalImage: true, + liveImage: false, + package: true, + }); + }); + it("plans the full release path against package-backed e2e images", () => { const plan = planFor({ includeOpenWebUI: false,