fix(docker): package selected external plugins in source-built images (#103629)

* build(docker): package selected external plugins

* test(docker): auto-clean plugin fixture

* fix(docker): preserve selected image provenance

* fix(docker): accept manifest-only plugin selections

* fix(docker): resolve selected plugin manifest ids

* fix(docker): isolate resolved plugin selection
This commit is contained in:
Peter Steinberger
2026-07-10 13:00:50 +01:00
committed by GitHub
parent 13819996ad
commit a238dead67
21 changed files with 807 additions and 62 deletions

View File

@@ -26,7 +26,10 @@ const livePackageBackedLanes = new Set([
]);
// These lanes intentionally build a focused source-checkout image instead of
// consuming the shared package E2E images.
const sourceCheckoutImageLanes = new Set(["plugin-binding-command-escape"]);
const sourceCheckoutImageLanes = new Set([
"docker-selected-plugins",
"plugin-binding-command-escape",
]);
function readText(relativePath) {
return fs.readFileSync(path.join(ROOT_DIR, relativePath), "utf8");

View File

@@ -0,0 +1,87 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "$ROOT_DIR/scripts/lib/docker-build.sh"
source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh"
IMAGE_NAME="${OPENCLAW_DOCKER_SELECTED_PLUGINS_E2E_IMAGE:-openclaw-docker-selected-plugins-e2e:local}"
DEPENDENCY_ONLY_IMAGE="${IMAGE_NAME}-dependency-only"
CONTAINER_NAME="openclaw-docker-selected-plugins-e2e-$$"
SELECTED_PLUGINS="${OPENCLAW_DOCKER_SELECTED_PLUGINS:-slack,msteams clickclack,slack}"
BUILD_GIT_COMMIT="${OPENCLAW_DOCKER_SELECTED_PLUGINS_E2E_GIT_COMMIT:-0123456789abcdef0123456789abcdef01234567}"
BUILD_TIMESTAMP="${OPENCLAW_DOCKER_SELECTED_PLUGINS_E2E_BUILD_TIMESTAMP:-2026-07-10T12:34:56.000Z}"
UNKNOWN_LOG="$(mktemp -t openclaw-docker-selected-plugins-unknown.XXXXXX)"
RUN_LOG="$(mktemp -t openclaw-docker-selected-plugins-run.XXXXXX)"
DOCKER_COMMAND_TIMEOUT="${OPENCLAW_DOCKER_SELECTED_PLUGINS_RUN_TIMEOUT:-900s}"
DEPENDENCY_ONLY_IMAGE_BUILT=0
cleanup() {
docker_e2e_docker_cmd rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
if [ "$DEPENDENCY_ONLY_IMAGE_BUILT" = "1" ]; then
docker_e2e_docker_cmd image rm -f "$DEPENDENCY_ONLY_IMAGE" >/dev/null 2>&1 || true
fi
rm -f "$UNKNOWN_LOG" "$RUN_LOG"
}
trap cleanup EXIT
if [ "${OPENCLAW_SKIP_DOCKER_BUILD:-0}" = "1" ]; then
echo "Reusing selected-plugin image: $IMAGE_NAME"
docker_e2e_docker_cmd image inspect "$IMAGE_NAME" >/dev/null
else
echo "Proving unknown selected plugins fail closed..."
set +e
docker_e2e_timeout_cmd "${OPENCLAW_DOCKER_SELECTED_PLUGINS_BUILD_TIMEOUT:-3600s}" \
env DOCKER_BUILDKIT=1 docker build \
--target workspace-deps \
--build-arg OPENCLAW_EXTENSIONS=missing-plugin \
-f "$ROOT_DIR/Dockerfile" \
"$ROOT_DIR" >"$UNKNOWN_LOG" 2>&1
unknown_status=$?
set -e
if [ "$unknown_status" -eq 0 ] || ! grep -Fq \
"unknown OPENCLAW_EXTENSIONS plugin id: missing-plugin" "$UNKNOWN_LOG"; then
echo "Unknown selected-plugin build did not fail closed as expected" >&2
docker_e2e_print_log "$UNKNOWN_LOG"
exit 1
fi
echo "Proving manifest ids and known dependency-only plugins remain stageable..."
docker_build_run docker-selected-plugins-dependency-only \
--target workspace-deps \
--build-arg OPENCLAW_EXTENSIONS=whatsapp,qqbot,kimi \
-t "$DEPENDENCY_ONLY_IMAGE" \
-f "$ROOT_DIR/Dockerfile" \
"$ROOT_DIR"
DEPENDENCY_ONLY_IMAGE_BUILT=1
docker_e2e_docker_run_cmd run --rm \
--entrypoint sh \
"$DEPENDENCY_ONLY_IMAGE" \
-c 'test -f /out/extensions/whatsapp/package.json && test -f /out/extensions/qqbot/package.json && test -f /out/extensions/kimi-coding/package.json && grep -qx kimi-coding /out/openclaw-selected-plugin-dirs'
echo "Building selected-plugin runtime image: $IMAGE_NAME"
docker_build_run docker-selected-plugins-build \
--build-arg "GIT_COMMIT=$BUILD_GIT_COMMIT" \
--build-arg "OPENCLAW_BUILD_TIMESTAMP=$BUILD_TIMESTAMP" \
--build-arg "OPENCLAW_EXTENSIONS=$SELECTED_PLUGINS" \
-t "$IMAGE_NAME" \
-f "$ROOT_DIR/Dockerfile" \
"$ROOT_DIR"
fi
echo "Inspecting selected plugins from the final runtime image..."
if ! docker_e2e_docker_run_cmd run --rm \
--name "$CONTAINER_NAME" \
--entrypoint bash \
-e "OPENCLAW_E2E_EXPECTED_GIT_COMMIT=$BUILD_GIT_COMMIT" \
-e "OPENCLAW_E2E_EXPECTED_BUILD_TIMESTAMP=$BUILD_TIMESTAMP" \
-v "$ROOT_DIR/scripts/e2e/lib/docker-selected-plugins:/openclaw-e2e:ro" \
"$IMAGE_NAME" \
/openclaw-e2e/scenario.sh >"$RUN_LOG" 2>&1; then
echo "Selected-plugin Docker E2E failed" >&2
docker_e2e_print_log "$RUN_LOG"
exit 1
fi
docker_e2e_print_log "$RUN_LOG"
echo "Selected-plugin Docker E2E passed"

View File

@@ -0,0 +1,118 @@
import fs from "node:fs";
import path from "node:path";
function assert(condition, message) {
if (!condition) {
throw new Error(message);
}
}
function assertFile(filePath) {
assert(fs.statSync(filePath).isFile(), `expected file: ${filePath}`);
}
function assertAbsent(filePath) {
assert(!fs.existsSync(filePath), `expected path to be absent: ${filePath}`);
}
function readJson(filePath) {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
}
const selected = {
clickclack: {
entries: ["index.js"],
capability: "channel",
},
slack: {
entries: ["index.js", "setup-entry.js"],
capability: "channel",
},
msteams: {
entries: ["index.js", "setup-entry.js"],
capability: "channel",
},
clawrouter: {
entries: ["index.js"],
capability: "text-inference",
},
};
const buildInfo = readJson("/app/dist/build-info.json");
const expectedCommit = process.env.OPENCLAW_E2E_EXPECTED_GIT_COMMIT?.toLowerCase();
const expectedBuiltAt = new Date(
process.env.OPENCLAW_E2E_EXPECTED_BUILD_TIMESTAMP ?? "",
).toISOString();
assert(buildInfo.commit === expectedCommit, `unexpected build commit: ${buildInfo.commit}`);
assert(buildInfo.builtAt === expectedBuiltAt, `unexpected build timestamp: ${buildInfo.builtAt}`);
for (const [pluginId, expected] of Object.entries(selected)) {
const pluginRoot = path.join("/app/dist/extensions", pluginId);
for (const entry of expected.entries) {
assertFile(path.join(pluginRoot, entry));
}
assertFile(path.join(pluginRoot, "openclaw.plugin.json"));
assertFile(path.join(pluginRoot, "package.json"));
const manifest = readJson(path.join(pluginRoot, "openclaw.plugin.json"));
const packageJson = readJson(path.join(pluginRoot, "package.json"));
assert(manifest.id === pluginId, `unexpected ${pluginId} manifest id: ${manifest.id}`);
assert(
packageJson.openclaw?.extensions?.includes("./index.js"),
`${pluginId} package entry was not rewritten to ./index.js`,
);
if (expected.entries.includes("setup-entry.js")) {
assert(
packageJson.openclaw?.setupEntry === "./setup-entry.js",
`${pluginId} setup entry was not rewritten to ./setup-entry.js`,
);
}
const inspect = readJson(`/tmp/openclaw-${pluginId}-inspect.json`);
assert(inspect.plugin?.id === pluginId, `unexpected ${pluginId} inspect id`);
assert(inspect.plugin?.status === "loaded", `${pluginId} runtime did not load`);
assert(inspect.plugin?.origin === "bundled", `${pluginId} did not load from bundled dist`);
assert(
inspect.capabilities?.some(
(entry) => entry?.kind === expected.capability && entry.ids?.includes(pluginId),
),
`${pluginId} did not register ${expected.capability} capability`,
);
}
for (const pluginId of ["clickclack", "slack"]) {
const packageJson = readJson(`/app/dist/extensions/${pluginId}/package.json`);
assert(
packageJson.openclaw?.build?.bundledDist === false,
`${pluginId} bundledDist release metadata changed`,
);
}
const declaredDependencies = {
clickclack: ["ws"],
slack: ["@slack/bolt", "@slack/web-api"],
msteams: ["@microsoft/teams.apps"],
};
for (const [pluginId, dependencies] of Object.entries(declaredDependencies)) {
const packageJson = readJson(`/app/dist/extensions/${pluginId}/package.json`);
for (const dependency of dependencies) {
assert(
typeof packageJson.dependencies?.[dependency] === "string",
`${pluginId} package metadata omitted ${dependency}`,
);
}
}
for (const dependency of ["@microsoft/teams.apps", "@slack/bolt", "@slack/web-api", "ws"]) {
assertFile(path.join("/app/node_modules", dependency, "package.json"));
}
assertFile("/app/dist/extensions/slack/skills/slack/SKILL.md");
assertAbsent("/app/dist/extensions/amazon-bedrock");
assertAbsent("/app/extensions/amazon-bedrock");
assertAbsent("/app/node_modules/@aws-sdk/client-bedrock");
assertAbsent("/app/dist/extensions/signal");
assertAbsent("/app/extensions/signal");
assertAbsent("/home/node/.cache/ms-playwright");
console.log(`Selected-plugin runtime proof passed (${process.arch})`);

View File

@@ -0,0 +1,28 @@
#!/usr/bin/env bash
set -euo pipefail
export HOME=/tmp/openclaw-docker-selected-plugins
export OPENCLAW_STATE_DIR="$HOME/.openclaw"
export OPENCLAW_CONFIG_PATH="$OPENCLAW_STATE_DIR/openclaw.json"
export OPENCLAW_DISABLE_BUNDLED_SOURCE_OVERLAYS=1
mkdir -p "$OPENCLAW_STATE_DIR"
node --input-type=module <<'NODE'
import fs from "node:fs";
const entries = Object.fromEntries(
["clickclack", "slack", "msteams"].map((id) => [id, { enabled: true }]),
);
fs.writeFileSync(
process.env.OPENCLAW_CONFIG_PATH,
`${JSON.stringify({ plugins: { entries } }, null, 2)}\n`,
{ mode: 0o600 },
);
NODE
for plugin_id in clickclack slack msteams clawrouter; do
node /app/openclaw.mjs plugins inspect "$plugin_id" --runtime --json \
>"/tmp/openclaw-${plugin_id}-inspect.json"
done
node /openclaw-e2e/assertions.mjs

View File

@@ -13,6 +13,7 @@ export type BundledPluginBuildEntryParams = {
};
export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS: Set<string>;
export const DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV: string;
export function collectRootPackageExcludedExtensionDirs(
params?: BundledPluginBuildEntryParams,
): Set<string>;

View File

@@ -14,6 +14,8 @@ const TOP_LEVEL_PUBLIC_SURFACE_EXTENSIONS = new Set([".ts", ".js", ".mts", ".cts
export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS = new Set(["qa-channel", "qa-lab", "qa-matrix"]);
const EXCLUDED_CORE_BUNDLED_PLUGIN_DIRS = new Set(["qqbot", "whatsapp"]);
const BUNDLED_PLUGIN_BUILD_IDS_ENV = "OPENCLAW_BUNDLED_PLUGIN_BUILD_IDS";
export const DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV = "OPENCLAW_INTERNAL_DOCKER_BUILD_PLUGIN_IDS";
const PLUGIN_ID_RE = /^[a-z0-9][a-z0-9-]*$/u;
const TOP_LEVEL_PRIVATE_TEST_SURFACE_RE =
/(?:^|[._-])(?:test|spec|test-support|test-helpers|test-fixtures|test-harness|mock-setup)(?:[._-]|$)/u;
const toPosixPath = (value) => value.replaceAll("\\", "/");
@@ -31,6 +33,28 @@ function parseBundledPluginBuildIdFilter(env = process.env) {
);
}
function parseDockerSelectedPluginBuildIdFilter(env = process.env) {
const raw = env[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV];
if (typeof raw !== "string" || raw.trim() === "") {
return null;
}
const ids = new Set(
raw
.split(/[\s,]+/u)
.map((entry) => entry.trim())
.filter((entry) => entry.length > 0),
);
const invalidIds = [...ids].filter((id) => !PLUGIN_ID_RE.test(id));
if (invalidIds.length > 0) {
throw new Error(
`${DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV} contains invalid plugin id(s): ${invalidIds
.toSorted((left, right) => left.localeCompare(right))
.join(", ")}`,
);
}
return ids;
}
function readBundledPluginPackageJson(packageJsonPath, options = {}) {
if (!(options.hasPackageJson ?? fs.existsSync(packageJsonPath))) {
return null;
@@ -184,7 +208,8 @@ function collectBundledPluginCandidates(cwd, extensionsRoot) {
relativeFiles: null,
topLevelPublicSurfaceEntries: collectTopLevelPublicSurfaceEntries(pluginDir),
};
});
})
.toSorted((left, right) => left.dirName.localeCompare(right.dirName));
}
/** Collect all bundled plugin build entries for the current checkout. */
@@ -192,9 +217,11 @@ export function collectBundledPluginBuildEntries(params = {}) {
const cwd = params.cwd ?? process.cwd();
const env = params.env ?? process.env;
const extensionsRoot = path.join(cwd, BUNDLED_PLUGIN_ROOT_DIR);
const dockerSelectedBuildIds = parseDockerSelectedPluginBuildIdFilter(env);
const candidates = collectBundledPluginCandidates(cwd, extensionsRoot);
const entries = [];
for (const candidate of collectBundledPluginCandidates(cwd, extensionsRoot)) {
for (const candidate of candidates) {
const { dirName, pluginDir, relativeFiles, topLevelPublicSurfaceEntries } = candidate;
const manifestPath = path.join(pluginDir, "openclaw.plugin.json");
const hasManifest =
@@ -216,7 +243,7 @@ export function collectBundledPluginBuildEntries(params = {}) {
if (!shouldBuildBundledCluster(dirName, env, { packageJson })) {
continue;
}
if (!shouldBuildBundledDistEntry(packageJson)) {
if (!shouldBuildBundledDistEntry(packageJson) && !dockerSelectedBuildIds?.has(dirName)) {
continue;
}
if (EXCLUDED_CORE_BUNDLED_PLUGIN_DIRS.has(dirName)) {
@@ -237,6 +264,18 @@ export function collectBundledPluginBuildEntries(params = {}) {
});
}
if (dockerSelectedBuildIds) {
const knownIds = new Set(candidates.map((candidate) => candidate.dirName));
const unknownIds = [...dockerSelectedBuildIds].filter((id) => !knownIds.has(id));
if (unknownIds.length > 0) {
throw new Error(
`${DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV} references unknown plugin id(s): ${unknownIds
.toSorted((left, right) => left.localeCompare(right))
.join(", ")}`,
);
}
}
const filteredBuildIds = parseBundledPluginBuildIdFilter(env);
if (!filteredBuildIds) {
return entries;

View File

@@ -262,15 +262,22 @@ function kitchenSinkRpcLane() {
}
export const mainLanes = [
serviceLane(
"compose-setup",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:compose-setup",
lane(
"docker-selected-plugins",
"OPENCLAW_SKIP_DOCKER_BUILD=0 pnpm test:docker:selected-plugins",
{
stateScenario: "empty",
timeoutMs: 20 * 60 * 1000,
weight: 3,
e2eImageKind: false,
estimateSeconds: 600,
resources: ["docker"],
timeoutMs: 30 * 60 * 1000,
weight: 4,
},
),
serviceLane("compose-setup", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:compose-setup", {
stateScenario: "empty",
timeoutMs: 20 * 60 * 1000,
weight: 3,
}),
npmLane(
"docker-package-install",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:package-install",
@@ -933,11 +940,10 @@ export function releasePathChunkLanes(chunk, options = {}) {
export function allReleasePathLanes(options = {}) {
const releaseProfile = normalizeReleaseProfile(options.releaseProfile);
return Object.keys(primaryReleasePathChunks)
.flatMap((chunk) =>
releasePathChunkLanes(chunk, {
includeOpenWebUI: options.includeOpenWebUI,
releaseProfile,
}),
);
return Object.keys(primaryReleasePathChunks).flatMap((chunk) =>
releasePathChunkLanes(chunk, {
includeOpenWebUI: options.includeOpenWebUI,
releaseProfile,
}),
);
}

View File

@@ -0,0 +1,83 @@
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
const PLUGIN_ID_RE = /^[a-z0-9][a-z0-9-]*$/u;
function readManifestId(pluginDir) {
const manifestPath = path.join(pluginDir, "openclaw.plugin.json");
if (!fs.existsSync(manifestPath)) {
return null;
}
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
return typeof manifest.id === "string" && manifest.id.length > 0 ? manifest.id : null;
}
function collectPluginIdentities(extensionsRoot) {
return fs
.readdirSync(extensionsRoot, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => {
const pluginDir = path.join(extensionsRoot, entry.name);
const hasPackageJson = fs.existsSync(path.join(pluginDir, "package.json"));
const manifestId = readManifestId(pluginDir);
return {
dirName: entry.name,
manifestId,
known: hasPackageJson || manifestId !== null,
};
})
.filter((entry) => entry.known)
.toSorted((left, right) => left.dirName.localeCompare(right.dirName));
}
/** Resolve public Docker selections to the source directories used by build and prune steps. */
export function resolveDockerPluginSelection(params) {
const selection = typeof params.selection === "string" ? params.selection : "";
const selectedIds = new Set(
selection
.split(/[\s,]+/u)
.map((entry) => entry.trim())
.filter(Boolean),
);
const plugins = collectPluginIdentities(params.extensionsRoot);
const resolvedDirs = new Set();
for (const selectedId of selectedIds) {
if (!PLUGIN_ID_RE.test(selectedId)) {
throw new Error(`invalid OPENCLAW_EXTENSIONS plugin id: ${selectedId}`);
}
const matches = plugins.filter(
(plugin) => plugin.dirName === selectedId || plugin.manifestId === selectedId,
);
if (matches.length === 0) {
throw new Error(`unknown OPENCLAW_EXTENSIONS plugin id: ${selectedId}`);
}
if (matches.length > 1) {
throw new Error(
`ambiguous OPENCLAW_EXTENSIONS plugin id: ${selectedId} (${matches
.map((plugin) => plugin.dirName)
.join(", ")})`,
);
}
resolvedDirs.add(matches[0].dirName);
}
return [...resolvedDirs].toSorted((left, right) => left.localeCompare(right));
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
try {
const resolved = resolveDockerPluginSelection({
extensionsRoot: process.argv[2],
selection: process.argv[3] ?? "",
});
if (resolved.length > 0) {
process.stdout.write(`${resolved.join("\n")}\n`);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error(`ERROR: ${message}`);
process.exitCode = 1;
}
}

View File

@@ -7,6 +7,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import * as tar from "tar";
import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "./lib/bundled-plugin-build-entries.mjs";
import { preparePackageChangelog, restorePackageChangelog } from "./package-changelog.mjs";
const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
@@ -22,6 +23,11 @@ const MAX_TIMER_TIMEOUT_MS = 2_147_000_000;
const AI_RUNTIME_PACKAGE = "@openclaw/ai";
const AI_RUNTIME_BACKUP_DIR = ".openclaw-ai-package-backup";
const ACTIVE_CHILD_KILLERS = new Set();
const PACKAGE_BUILD_PLUGIN_SELECTION_ENV_NAMES = [
"OPENCLAW_EXTENSIONS",
"OPENCLAW_DOCKER_BUILD_EXTENSIONS",
DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV,
];
const SIGNAL_EXIT_CODES = {
SIGHUP: 129,
SIGINT: 130,
@@ -365,13 +371,19 @@ const PACKAGE_ARTIFACT_BUILD_STEPS = [
export async function buildPackageArtifacts(sourceDir, options = {}) {
const runImpl = options.runImpl ?? run;
const buildEnv = {
...process.env,
OPENCLAW_BUILD_ALL_NO_PNPM: "1",
OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "0",
};
for (const envName of PACKAGE_BUILD_PLUGIN_SELECTION_ENV_NAMES) {
delete buildEnv[envName];
}
for (const step of PACKAGE_ARTIFACT_BUILD_STEPS) {
console.error(`==> ${step.label}`);
await runImpl(step.command, step.args, sourceDir, {
env: {
...process.env,
OPENCLAW_BUILD_ALL_NO_PNPM: "1",
OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "0",
...buildEnv,
},
timeoutMs: resolveTimeoutMs(
"OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS",

View File

@@ -73,6 +73,9 @@ type ReleaseCheckCommandInvocation = {
};
const rootPackageExcludedExtensionDirs = collectRootPackageExcludedExtensionDirs();
const rootPackageExcludedExtensionPrefixes = [...rootPackageExcludedExtensionDirs].map(
(extensionId) => `dist/extensions/${extensionId}/`,
);
const requiredPathGroups = [
"npm-shrinkwrap.json",
PACKAGE_DIST_INVENTORY_RELATIVE_PATH,
@@ -105,6 +108,7 @@ const requiredPathGroups = [
"dist/control-ui/index.html",
];
const forbiddenPrefixes = [
...rootPackageExcludedExtensionPrefixes,
...LOCAL_BUILD_METADATA_DIST_PATHS,
"dist-runtime/",
"dist/OpenClaw.app/",