From a749939adefef1affeb9de7add0700111a667788 Mon Sep 17 00:00:00 2001 From: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:22:19 +0000 Subject: [PATCH] fix(ci): package runtime resources in dist artifact --- .github/workflows/ci.yml | 16 +- scripts/dist-runtime-build-artifact.mjs | 13 + .../dist-runtime-artifact-resolver-hook.mjs | 28 ++ scripts/lib/workspace-bootstrap-smoke.d.mts | 19 +- scripts/lib/workspace-bootstrap-smoke.mjs | 318 +++++++++++++++++- test/scripts/ci-workflow-guards.test.ts | 67 +++- 6 files changed, 450 insertions(+), 11 deletions(-) create mode 100644 scripts/dist-runtime-build-artifact.mjs create mode 100644 scripts/lib/dist-runtime-artifact-resolver-hook.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d288791d898d..dc865b082193 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1134,13 +1134,25 @@ jobs: fi - name: Pack built runtime artifacts - run: tar --posix -cf dist-runtime-build.tar.zst --use-compress-program zstdmt dist dist-runtime packages/*/dist + env: + COMPATIBILITY_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + run: | + set -euo pipefail + if [[ -f scripts/dist-runtime-build-artifact.mjs ]]; then + node scripts/dist-runtime-build-artifact.mjs "$RUNNER_TEMP/dist-runtime-build.tar.zst" + elif [[ "$COMPATIBILITY_TARGET" == "true" ]]; then + echo "Selected compatibility target predates the runtime artifact manifest; using its legacy build outputs." + tar --posix -cf "$RUNNER_TEMP/dist-runtime-build.tar.zst" --use-compress-program zstdmt dist dist-runtime packages/*/dist + else + echo "Current CI target is missing scripts/dist-runtime-build-artifact.mjs" >&2 + exit 1 + fi - name: Upload built runtime artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: dist-runtime-build - path: dist-runtime-build.tar.zst + path: ${{ runner.temp }}/dist-runtime-build.tar.zst retention-days: 1 - name: Upload bundled plugin asset artifacts diff --git a/scripts/dist-runtime-build-artifact.mjs b/scripts/dist-runtime-build-artifact.mjs new file mode 100644 index 000000000000..a93028e65d0e --- /dev/null +++ b/scripts/dist-runtime-build-artifact.mjs @@ -0,0 +1,13 @@ +import path from "node:path"; +import { buildAndSmokeDistRuntimeArtifact } from "./lib/workspace-bootstrap-smoke.mjs"; + +const archivePath = process.argv[2]; +if (!archivePath) { + throw new Error("Usage: node scripts/dist-runtime-build-artifact.mjs "); +} + +const result = await buildAndSmokeDistRuntimeArtifact({ + rootDir: process.cwd(), + archivePath: path.resolve(archivePath), +}); +console.log(`dist runtime artifact ready: ${result.archivePath}`); diff --git a/scripts/lib/dist-runtime-artifact-resolver-hook.mjs b/scripts/lib/dist-runtime-artifact-resolver-hook.mjs new file mode 100644 index 000000000000..4a0c767667fc --- /dev/null +++ b/scripts/lib/dist-runtime-artifact-resolver-hook.mjs @@ -0,0 +1,28 @@ +import { registerHooks } from "node:module"; +import { resolveDistRuntimeArtifactWorkspaceImport } from "./workspace-bootstrap-smoke.mjs"; + +const artifactRoot = process.env.OPENCLAW_ARTIFACT_ROOT; +if (!artifactRoot) { + throw new Error("OPENCLAW_ARTIFACT_ROOT is required for the runtime artifact smoke"); +} +const workspacePackagesJson = process.env.OPENCLAW_ARTIFACT_WORKSPACE_PACKAGES; +if (!workspacePackagesJson) { + throw new Error( + "OPENCLAW_ARTIFACT_WORKSPACE_PACKAGES is required for the runtime artifact smoke", + ); +} +const workspacePackageNames = new Set(JSON.parse(workspacePackagesJson)); + +registerHooks({ + resolve(specifier, context, nextResolve) { + const url = resolveDistRuntimeArtifactWorkspaceImport({ + artifactRoot, + specifier, + workspacePackageNames, + }); + if (url) { + return { shortCircuit: true, url }; + } + return nextResolve(specifier, context); + }, +}); diff --git a/scripts/lib/workspace-bootstrap-smoke.d.mts b/scripts/lib/workspace-bootstrap-smoke.d.mts index 1d1398a7a6c4..d6f9cf4fc047 100644 --- a/scripts/lib/workspace-bootstrap-smoke.d.mts +++ b/scripts/lib/workspace-bootstrap-smoke.d.mts @@ -1,7 +1,24 @@ export const WORKSPACE_TEMPLATE_PACK_PATHS: readonly string[]; +export const DIST_RUNTIME_ARTIFACT_BASE_PATHS: readonly string[]; +export const DIST_RUNTIME_ARTIFACT_WORKSPACE_PACKAGE_NAMES: readonly string[]; export function createWorkspaceBootstrapSmokeEnv( env: NodeJS.ProcessEnv, homeDir: string, overrides?: NodeJS.ProcessEnv, ): NodeJS.ProcessEnv; -export function runInstalledWorkspaceBootstrapSmoke(params: { packageRoot: string }): void; +export function runInstalledWorkspaceBootstrapSmoke(params: { + packageRoot: string; + nodeArgs?: string[]; + envOverrides?: NodeJS.ProcessEnv; +}): void; +export function collectDistRuntimeArtifactPaths(rootDir: string): string[]; +export function resolveDistRuntimeArtifactWorkspaceImport(params: { + artifactRoot: string; + specifier: string; + workspacePackageNames: ReadonlySet; +}): string | undefined; +export function buildAndSmokeDistRuntimeArtifact(params: { + rootDir: string; + archivePath: string; + compressor?: string; +}): Promise<{ archivePath: string; artifactPaths: string[] }>; diff --git a/scripts/lib/workspace-bootstrap-smoke.mjs b/scripts/lib/workspace-bootstrap-smoke.mjs index 4afb049889e6..04e41988af36 100644 --- a/scripts/lib/workspace-bootstrap-smoke.mjs +++ b/scripts/lib/workspace-bootstrap-smoke.mjs @@ -1,8 +1,11 @@ // Verifies installed packages can bootstrap the default OpenClaw workspace files. -import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { execFileSync, spawn } from "node:child_process"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { createServer } from "node:net"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { TSDOWN_PACKAGE_OUTPUT_ROOTS } from "./tsdown-output-roots.mjs"; /** * Template pack files that must be present in installed packages. @@ -16,6 +19,23 @@ export const WORKSPACE_TEMPLATE_PACK_PATHS = [ "docs/reference/templates/BOOTSTRAP.md", ]; +export const DIST_RUNTIME_ARTIFACT_BASE_PATHS = [ + "openclaw.mjs", + "package.json", + "docs/reference/templates", + "src/agents/templates", + "dist", + "dist-runtime", +]; + +const DIST_RUNTIME_ARTIFACT_PACKAGE_DIST_PATHS = [ + ...TSDOWN_PACKAGE_OUTPUT_ROOTS, + "packages/plugin-sdk/dist", +].toSorted((left, right) => left.localeCompare(right)); + +export const DIST_RUNTIME_ARTIFACT_WORKSPACE_PACKAGE_NAMES = + DIST_RUNTIME_ARTIFACT_PACKAGE_DIST_PATHS.map((distPath) => `@openclaw/${distPath.split("/")[1]}`); + // HEARTBEAT.md ships in the template pack for docs/doctor context but is no // longer seeded into new workspaces; heartbeat context lives in cron scratch. const REQUIRED_BOOTSTRAP_WORKSPACE_FILES = [ @@ -27,7 +47,12 @@ const REQUIRED_BOOTSTRAP_WORKSPACE_FILES = [ ]; const WORKSPACE_BOOTSTRAP_SMOKE_TIMEOUT_MS = 15_000; +const DIST_RUNTIME_ARTIFACT_SMOKE_TIMEOUT_MS = 20_000; +const DIST_RUNTIME_ARTIFACT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; const SAFE_UNIX_SMOKE_PATH = "/usr/bin:/bin"; +const DIST_RUNTIME_ARTIFACT_RESOLVER_HOOK = fileURLToPath( + new URL("./dist-runtime-artifact-resolver-hook.mjs", import.meta.url), +); /** * Creates a minimal isolated environment for workspace bootstrap smoke runs. @@ -112,6 +137,7 @@ export function runInstalledWorkspaceBootstrapSmoke(params) { execFileSync( process.execPath, [ + ...(params.nodeArgs ?? []), join(params.packageRoot, "openclaw.mjs"), "agent", "--message", @@ -129,7 +155,7 @@ export function runInstalledWorkspaceBootstrapSmoke(params) { maxBuffer: 1024 * 1024 * 16, stdio: ["ignore", "pipe", "pipe"], timeout: WORKSPACE_BOOTSTRAP_SMOKE_TIMEOUT_MS, - env: createWorkspaceBootstrapSmokeEnv(process.env, homeDir), + env: createWorkspaceBootstrapSmokeEnv(process.env, homeDir, params.envOverrides), }, ); } catch (error) { @@ -158,3 +184,287 @@ export function runInstalledWorkspaceBootstrapSmoke(params) { } } } + +export function collectDistRuntimeArtifactPaths(rootDir) { + const packagePaths = DIST_RUNTIME_ARTIFACT_PACKAGE_DIST_PATHS.flatMap((distPath) => { + const packageRoot = dirname(distPath); + return [`${packageRoot}/package.json`, distPath]; + }); + const missingPaths = packagePaths.filter( + (artifactPath) => !existsSync(join(rootDir, artifactPath)), + ); + if (missingPaths.length > 0) { + throw new Error( + `dist runtime artifact inputs are missing required workspace package paths: ${missingPaths.join(", ")}`, + ); + } + return [...DIST_RUNTIME_ARTIFACT_BASE_PATHS, ...packagePaths].toSorted((left, right) => + left.localeCompare(right), + ); +} + +function selectRuntimeExport(value) { + if (typeof value === "string") { + return value; + } + if (!value || typeof value !== "object") { + return undefined; + } + for (const condition of ["import", "node", "default"]) { + const selected = selectRuntimeExport(value[condition]); + if (selected) { + return selected; + } + } + return undefined; +} + +export function resolveDistRuntimeArtifactWorkspaceImport(params) { + if (!params.specifier.startsWith("@openclaw/")) { + return undefined; + } + const parts = params.specifier.split("/"); + const packageName = parts.slice(0, 2).join("/"); + if (!params.workspacePackageNames.has(packageName)) { + return undefined; + } + + const packageRoot = join(params.artifactRoot, "packages", parts[1]); + const packageJsonPath = join(packageRoot, "package.json"); + if (!existsSync(packageJsonPath)) { + throw new Error(`artifact workspace package is unavailable: ${packageName}`); + } + const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")); + const exportName = parts.length > 2 ? `./${parts.slice(2).join("/")}` : "."; + const target = selectRuntimeExport(packageJson.exports?.[exportName]); + if (!target || target.includes("/src/")) { + throw new Error(`artifact workspace export is unavailable: ${params.specifier}`); + } + return pathToFileURL(resolve(packageRoot, target)).href; +} + +function listDistRuntimeArtifactEntries(archivePath, compressor) { + return execFileSync("tar", ["--use-compress-program", compressor, "-tf", archivePath], { + encoding: "utf8", + maxBuffer: DIST_RUNTIME_ARTIFACT_MAX_OUTPUT_BYTES, + }) + .split(/\r?\n/u) + .map((entry) => entry.replace(/^\.\//u, "").replace(/\/$/u, "")) + .filter(Boolean); +} + +function validateDistRuntimeArtifactEntries(entries, expectedPaths) { + const entrySet = new Set(entries); + const missingPaths = expectedPaths.filter( + (expectedPath) => + !entrySet.has(expectedPath) && !entries.some((entry) => entry.startsWith(`${expectedPath}/`)), + ); + if (missingPaths.length > 0) { + throw new Error(`dist runtime artifact is missing required paths: ${missingPaths.join(", ")}`); + } + + const hasPluginRuntime = entries.some( + (entry) => + entry.startsWith("dist-runtime/extensions/") && entry.endsWith("/openclaw.plugin.json"), + ); + if (!hasPluginRuntime) { + throw new Error("dist runtime artifact is missing the built plugin runtime layout"); + } + + const unexpectedEntries = entries.filter((entry) => { + if (entry === "openclaw.mjs" || entry === "package.json") { + return false; + } + if ( + entry === "dist" || + entry.startsWith("dist/") || + entry === "dist-runtime" || + entry.startsWith("dist-runtime/") || + entry === "src" || + entry === "src/agents" || + entry === "src/agents/templates" || + entry.startsWith("src/agents/templates/") || + entry === "docs" || + entry === "docs/reference" || + entry === "docs/reference/templates" || + entry.startsWith("docs/reference/templates/") || + entry === "packages" + ) { + return false; + } + return !/^packages\/[^/]+\/(?:package\.json|dist(?:\/.*)?)$/u.test(entry); + }); + if (unexpectedEntries.length > 0) { + throw new Error( + `dist runtime artifact contains paths outside the runtime allowlist: ${unexpectedEntries.slice(0, 10).join(", ")}`, + ); + } +} + +function reserveLoopbackPort() { + return new Promise((resolvePromise, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + reject(new Error("failed to reserve a loopback port for the gateway smoke")); + return; + } + server.close((error) => { + if (error) { + reject(new Error("failed to release the reserved gateway smoke port", { cause: error })); + return; + } + resolvePromise(address.port); + }); + }); + }); +} + +async function waitForGatewayReadiness(params) { + const deadline = Date.now() + DIST_RUNTIME_ARTIFACT_SMOKE_TIMEOUT_MS; + while (Date.now() < deadline) { + if (params.child.exitCode !== null) { + throw new Error(`extracted gateway exited before readiness:\n${params.readOutput()}`); + } + try { + const response = await fetch(`http://127.0.0.1:${params.port}/readyz`, { + signal: AbortSignal.timeout(1_000), + }); + if (response.ok) { + const body = await response.json(); + if (body?.ready === true) { + return; + } + } + } catch { + // Gateway startup is asynchronous; retry until the bounded deadline. + } + await new Promise((resolvePromise) => { + setTimeout(resolvePromise, 250); + }); + } + throw new Error(`extracted gateway did not become ready:\n${params.readOutput()}`); +} + +async function stopGatewaySmoke(child) { + if (child.exitCode !== null) { + return; + } + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolvePromise) => { + child.once("exit", resolvePromise); + }), + new Promise((resolvePromise) => { + setTimeout(resolvePromise, 5_000); + }), + ]); + if (child.exitCode === null) { + child.kill("SIGKILL"); + } +} + +export async function buildAndSmokeDistRuntimeArtifact(params) { + const rootDir = resolve(params.rootDir); + const archivePath = resolve(params.archivePath); + const compressor = params.compressor ?? "zstdmt"; + const artifactPaths = collectDistRuntimeArtifactPaths(rootDir); + mkdirSync(dirname(archivePath), { recursive: true }); + execFileSync( + "tar", + ["--posix", "-cf", archivePath, "--use-compress-program", compressor, ...artifactPaths], + { cwd: rootDir, stdio: "inherit" }, + ); + + const entries = listDistRuntimeArtifactEntries(archivePath, compressor); + validateDistRuntimeArtifactEntries(entries, artifactPaths); + + const smokeParent = join(rootDir, ".artifacts"); + mkdirSync(smokeParent, { recursive: true }); + const smokeRoot = mkdtempSync(join(smokeParent, "dist-runtime-artifact-smoke-")); + const packageRoot = join(smokeRoot, "package"); + const homeDir = join(smokeRoot, "home"); + const cwd = join(smokeRoot, "cwd"); + mkdirSync(packageRoot, { recursive: true }); + mkdirSync(homeDir, { recursive: true }); + mkdirSync(cwd, { recursive: true }); + + const nodeArgs = ["--import", DIST_RUNTIME_ARTIFACT_RESOLVER_HOOK]; + const artifactEnvOverrides = { + OPENCLAW_ARTIFACT_ROOT: packageRoot, + OPENCLAW_ARTIFACT_WORKSPACE_PACKAGES: JSON.stringify( + DIST_RUNTIME_ARTIFACT_WORKSPACE_PACKAGE_NAMES, + ), + OPENCLAW_CONFIG_PATH: join(homeDir, "openclaw.json"), + OPENCLAW_STATE_DIR: join(homeDir, "state"), + }; + const smokeEnv = createWorkspaceBootstrapSmokeEnv(process.env, homeDir, artifactEnvOverrides); + + let gateway; + let gatewayOutput = ""; + try { + execFileSync( + "tar", + ["--use-compress-program", compressor, "-xf", archivePath, "-C", packageRoot], + { stdio: "inherit" }, + ); + runInstalledWorkspaceBootstrapSmoke({ + packageRoot, + nodeArgs, + envOverrides: artifactEnvOverrides, + }); + + const acpHelp = execFileSync( + process.execPath, + [...nodeArgs, join(packageRoot, "openclaw.mjs"), "acp", "--help"], + { + cwd, + encoding: "utf8", + env: smokeEnv, + maxBuffer: DIST_RUNTIME_ARTIFACT_MAX_OUTPUT_BYTES, + timeout: DIST_RUNTIME_ARTIFACT_SMOKE_TIMEOUT_MS, + }, + ); + if (!acpHelp.includes("Usage: openclaw acp")) { + throw new Error("extracted ACP startup did not return the expected help output"); + } + + const port = await reserveLoopbackPort(); + gateway = spawn( + process.execPath, + [ + ...nodeArgs, + join(packageRoot, "openclaw.mjs"), + "gateway", + "run", + "--allow-unconfigured", + "--auth", + "none", + "--bind", + "loopback", + "--port", + String(port), + ], + { cwd, env: smokeEnv, stdio: ["ignore", "pipe", "pipe"] }, + ); + const appendGatewayOutput = (chunk) => { + gatewayOutput += chunk.toString(); + if (Buffer.byteLength(gatewayOutput) > DIST_RUNTIME_ARTIFACT_MAX_OUTPUT_BYTES) { + gatewayOutput = gatewayOutput.slice(-DIST_RUNTIME_ARTIFACT_MAX_OUTPUT_BYTES); + } + }; + gateway.stdout.on("data", appendGatewayOutput); + gateway.stderr.on("data", appendGatewayOutput); + await waitForGatewayReadiness({ child: gateway, port, readOutput: () => gatewayOutput }); + } finally { + if (gateway) { + await stopGatewaySmoke(gateway); + } + rmSync(smokeRoot, { force: true, recursive: true }); + } + + return { archivePath, artifactPaths }; +} diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 5e62f5f487a7..f5b39d381a0e 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -17,6 +17,7 @@ import path from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; import { describe, expect, it } from "vitest"; import { parse } from "yaml"; +import { resolveDistRuntimeArtifactWorkspaceImport } from "../../scripts/lib/workspace-bootstrap-smoke.mjs"; import { NATIVE_I18N_LOCALES } from "../../scripts/native-app-i18n.ts"; import { SUPPORTED_LOCALES } from "../../ui/src/i18n/lib/registry.ts"; @@ -4655,10 +4656,46 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(restoreStep.with.path).toContain("packages/*/dist/"); expect(saveStep.with.path).toContain("packages/*/dist/"); expect(restoreStep.with.key).toContain("dist-build-v3-"); - expect( - buildArtifactSteps.find((step: WorkflowStep) => step.name === "Pack built runtime artifacts") - .run, - ).toContain("packages/*/dist"); + const packStep = buildArtifactSteps.find( + (step: WorkflowStep) => step.name === "Pack built runtime artifacts", + ); + const uploadStep = buildArtifactSteps.find( + (step: WorkflowStep) => step.name === "Upload built runtime artifacts", + ); + expect(packStep.env.COMPATIBILITY_TARGET).toBe( + "${{ needs.preflight.outputs.compatibility_target }}", + ); + expect(packStep.run).toContain("[[ -f scripts/dist-runtime-build-artifact.mjs ]]"); + expect(packStep.run).toContain( + 'node scripts/dist-runtime-build-artifact.mjs "$RUNNER_TEMP/dist-runtime-build.tar.zst"', + ); + expect(packStep.run).toContain('[[ "$COMPATIBILITY_TARGET" == "true" ]]'); + expect(packStep.run).toContain("predates the runtime artifact manifest"); + expect(packStep.run).toContain( + 'tar --posix -cf "$RUNNER_TEMP/dist-runtime-build.tar.zst" --use-compress-program zstdmt dist dist-runtime packages/*/dist', + ); + expect(packStep.run).toContain( + "Current CI target is missing scripts/dist-runtime-build-artifact.mjs", + ); + expect(uploadStep.with.path).toBe("${{ runner.temp }}/dist-runtime-build.tar.zst"); + const artifactBuilder = readFileSync("scripts/lib/workspace-bootstrap-smoke.mjs", "utf8"); + for (const requiredPath of [ + '"openclaw.mjs"', + '"package.json"', + '"docs/reference/templates"', + '"src/agents/templates"', + '"dist"', + '"dist-runtime"', + "TSDOWN_PACKAGE_OUTPUT_ROOTS", + '"packages/plugin-sdk/dist"', + "DIST_RUNTIME_ARTIFACT_WORKSPACE_PACKAGE_NAMES", + ]) { + expect(artifactBuilder).toContain(requiredPath); + } + expect(artifactBuilder).toContain("runInstalledWorkspaceBootstrapSmoke"); + expect(artifactBuilder).toContain('"acp", "--help"'); + expect(artifactBuilder).toContain("/readyz"); + expect(artifactBuilder).toContain("dist-runtime/extensions/"); expect(restoreStep.with.path).toContain("extensions/*/src/host/**/.bundle.hash"); expect(restoreStep.with.path).toContain("extensions/*/src/host/**/*.bundle.js"); expect(buildArtifactSteps.map((step: WorkflowStep) => step.name)).not.toContain( @@ -4666,6 +4703,28 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" ); }); + it("fails closed when an extracted workspace package is absent", () => { + const artifactRoot = mkdtempSync(path.join(tmpdir(), "openclaw-runtime-artifact-")); + try { + expect(() => + resolveDistRuntimeArtifactWorkspaceImport({ + artifactRoot, + specifier: "@openclaw/ai", + workspacePackageNames: new Set(["@openclaw/ai"]), + }), + ).toThrow("artifact workspace package is unavailable: @openclaw/ai"); + expect( + resolveDistRuntimeArtifactWorkspaceImport({ + artifactRoot, + specifier: "@openclaw/fs-safe", + workspacePackageNames: new Set(["@openclaw/ai"]), + }), + ).toBeUndefined(); + } finally { + rmSync(artifactRoot, { force: true, recursive: true }); + } + }); + it("keeps the AI runtime in Testbox build artifact caches", () => { const workflow = readBuildArtifactsTestboxWorkflow(); const steps = workflow.jobs["build-artifacts"].steps;