diff --git a/Dockerfile b/Dockerfile index 81ff1df71f05..5251a47842a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -# Opt-in plugin dependencies at build time (space- or comma-separated directory names). +# Opt-in plugin dependencies and supported runtime builds (space- or comma-separated ids). +# Manifest ids and existing source-directory names are accepted. # Example: docker build --build-arg OPENCLAW_EXTENSIONS="diagnostics-otel,matrix" . # # Multi-stage build produces a minimal runtime image without build tools, @@ -30,8 +31,10 @@ FROM ${OPENCLAW_NODE_BOOKWORM_IMAGE} AS workspace-deps ARG OPENCLAW_EXTENSIONS ARG OPENCLAW_BUNDLED_PLUGIN_DIR # Copy package.json files for workspace packages used by the install layer. +# Manifest-only bundled plugins remain valid selections but need no workspace metadata. RUN --mount=type=bind,source=packages,target=/tmp/packages,readonly \ --mount=type=bind,source=${OPENCLAW_BUNDLED_PLUGIN_DIR},target=/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR},readonly \ + --mount=type=bind,source=scripts/lib/docker-plugin-selection.mjs,target=/tmp/docker-plugin-selection.mjs,readonly \ mkdir -p /out/packages "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}" && \ for manifest in /tmp/packages/*/package.json; do \ [ -f "$manifest" ] || continue; \ @@ -40,18 +43,20 @@ RUN --mount=type=bind,source=packages,target=/tmp/packages,readonly \ mkdir -p "/out/packages/$pkg_name" && \ cp "$manifest" "/out/packages/$pkg_name/package.json"; \ done && \ - for ext in $(printf '%s\n' "$OPENCLAW_EXTENSIONS" | tr ',' ' '); do \ - if [ -f "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext/package.json" ]; then \ + node /tmp/docker-plugin-selection.mjs "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}" "$OPENCLAW_EXTENSIONS" \ + > /out/openclaw-selected-plugin-dirs && \ + while IFS= read -r ext; do \ + ext_dir="/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext"; \ + if [ -f "$ext_dir/package.json" ]; then \ mkdir -p "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext" && \ - cp "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext/package.json" "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext/package.json"; \ + cp "$ext_dir/package.json" "/out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext/package.json"; \ fi; \ - done + done < /out/openclaw-selected-plugin-dirs # ── Stage 2: Build ────────────────────────────────────────────── FROM ${OPENCLAW_BUN_IMAGE} AS bun-binary FROM ${OPENCLAW_NODE_BOOKWORM_IMAGE} AS build ARG OPENCLAW_BUNDLED_PLUGIN_DIR -ARG OPENCLAW_EXTENSIONS ARG OPENCLAW_DOCKER_BUILD_NODE_OPTIONS ARG OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB ARG OPENCLAW_DOCKER_BUILD_SKIP_DTS @@ -72,6 +77,7 @@ COPY scripts/lib/package-dist-imports.mjs ./scripts/lib/package-dist-imports.mjs COPY --from=workspace-deps /out/packages/ ./packages/ COPY --from=workspace-deps /out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/ ./${OPENCLAW_BUNDLED_PLUGIN_DIR}/ +COPY --from=workspace-deps /out/openclaw-selected-plugin-dirs /tmp/openclaw-selected-plugin-dirs # Reduce OOM risk on low-memory hosts during dependency installation. # Docker builds on small VMs may otherwise fail with "Killed" (exit 137). @@ -86,7 +92,7 @@ RUN --mount=type=cache,id=openclaw-pnpm-store,target=/root/.local/share/pnpm/sto # still exiting successfully, so retry the package downloader before failing. # Skip the entire check when matrix is not a bundled extension (e.g. msteams-only builds). RUN set -eux; \ - if ! printf '%s\n' "$OPENCLAW_EXTENSIONS" | tr ',' ' ' | tr ' ' '\n' | grep -qx 'matrix'; then \ + if ! grep -qx 'matrix' /tmp/openclaw-selected-plugin-dirs; then \ echo "==> matrix not bundled, skipping matrix-sdk-crypto check"; \ exit 0; \ fi; \ @@ -132,16 +138,17 @@ RUN pnpm_config_verify_deps_before_run=false pnpm canvas:a2ui:bundle || \ # Force pnpm for UI build (Bun may fail on ARM/Synology architectures) ENV OPENCLAW_PREFER_PNPM=1 RUN set -eu; \ + selected_plugin_dirs="$(cat /tmp/openclaw-selected-plugin-dirs)"; \ if [ -z "$OPENCLAW_BUILD_TIMESTAMP" ]; then \ OPENCLAW_BUILD_TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"; \ export OPENCLAW_BUILD_TIMESTAMP; \ fi; \ - if printf '%s\n' "$OPENCLAW_EXTENSIONS" | tr ',' ' ' | tr ' ' '\n' | grep -qx 'qa-lab'; then \ + if grep -qx 'qa-lab' /tmp/openclaw-selected-plugin-dirs; then \ export OPENCLAW_BUILD_PRIVATE_QA=1 OPENCLAW_ENABLE_PRIVATE_QA_CLI=1; \ fi; \ - OPENCLAW_RUN_NODE_SKIP_DTS_BUILD="$OPENCLAW_DOCKER_BUILD_SKIP_DTS" OPENCLAW_TSDOWN_MAX_OLD_SPACE_MB="$OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB" NODE_OPTIONS="$OPENCLAW_DOCKER_BUILD_NODE_OPTIONS" pnpm_config_verify_deps_before_run=false pnpm build:docker; \ + OPENCLAW_INTERNAL_DOCKER_BUILD_PLUGIN_IDS="$selected_plugin_dirs" OPENCLAW_RUN_NODE_SKIP_DTS_BUILD="$OPENCLAW_DOCKER_BUILD_SKIP_DTS" OPENCLAW_TSDOWN_MAX_OLD_SPACE_MB="$OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB" NODE_OPTIONS="$OPENCLAW_DOCKER_BUILD_NODE_OPTIONS" pnpm_config_verify_deps_before_run=false pnpm build:docker; \ pnpm_config_verify_deps_before_run=false pnpm ui:build -RUN if printf '%s\n' "$OPENCLAW_EXTENSIONS" | tr ',' ' ' | tr ' ' '\n' | grep -qx 'qa-lab'; then \ +RUN if grep -qx 'qa-lab' /tmp/openclaw-selected-plugin-dirs; then \ pnpm_config_verify_deps_before_run=false pnpm qa:lab:build && \ mkdir -p dist/extensions/qa-lab/web && \ rm -rf dist/extensions/qa-lab/web/dist && \ @@ -151,7 +158,6 @@ RUN if printf '%s\n' "$OPENCLAW_EXTENSIONS" | tr ',' ' ' | tr ' ' '\n' | grep -q # Prune dev dependencies, omitted plugin runtime packages, and build-only # metadata before copying runtime assets into the final image. FROM build AS runtime-assets -ARG OPENCLAW_EXTENSIONS ARG OPENCLAW_BUNDLED_PLUGIN_DIR # BuildKit cache mounts are not part of cached layers; seed tarballs for the # installed prod graph in the same step that runs offline prune. @@ -162,7 +168,7 @@ RUN --mount=type=cache,id=openclaw-pnpm-store,target=/root/.local/share/pnpm/sto --config.supportedArchitectures.os=linux \ --config.supportedArchitectures.cpu="$(node -p 'process.arch')" \ --config.supportedArchitectures.libc=glibc && \ - OPENCLAW_EXTENSIONS="$OPENCLAW_EXTENSIONS" OPENCLAW_BUNDLED_PLUGIN_DIR="$OPENCLAW_BUNDLED_PLUGIN_DIR" node scripts/prune-docker-plugin-dist.mjs && \ + OPENCLAW_EXTENSIONS="$(cat /tmp/openclaw-selected-plugin-dirs)" OPENCLAW_BUNDLED_PLUGIN_DIR="$OPENCLAW_BUNDLED_PLUGIN_DIR" node scripts/prune-docker-plugin-dist.mjs && \ node scripts/postinstall-bundled-plugins.mjs && \ find dist -type f \( -name '*.d.ts' -o -name '*.d.mts' -o -name '*.d.cts' -o -name '*.map' \) -delete && \ if [ -L /app/node_modules/@openclaw/ai ]; then \ diff --git a/docs/docs_map.md b/docs/docs_map.md index 7963491bab52..362834350592 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -4216,6 +4216,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H3: Manual flow - H3: Upgrading container images - H3: Environment variables + - H3: Source-built images with selected plugins - H3: Observability - H3: Health checks - H3: LAN vs loopback diff --git a/docs/install/docker.md b/docs/install/docker.md index d2206cc64f24..06dc18cac6c4 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -151,29 +151,29 @@ same PVC, then restart the Deployment or StatefulSet. Optional variables accepted by `scripts/docker/setup.sh` (and, for the gateway container, by `docker-compose.yml` directly): -| Variable | Purpose | -| ----------------------------------------------- | ------------------------------------------------------------------------------------------------------- | -| `OPENCLAW_IMAGE` | Use a remote image instead of building locally | -| `OPENCLAW_IMAGE_APT_PACKAGES` | Install extra apt packages during build (space-separated). Legacy alias: `OPENCLAW_DOCKER_APT_PACKAGES` | -| `OPENCLAW_IMAGE_PIP_PACKAGES` | Install extra Python packages during build (space-separated) | -| `OPENCLAW_EXTENSIONS` | Pre-install plugin dependencies at build time (comma- or space-separated ids) | -| `OPENCLAW_DOCKER_BUILD_NODE_OPTIONS` | Override the local source-build Node options (default `--max-old-space-size=8192`) | -| `OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB` | Override the local source-build tsdown heap in MB | -| `OPENCLAW_DOCKER_BUILD_SKIP_DTS` | Skip declaration output during runtime-only local image builds (default `1`) | -| `OPENCLAW_INSTALL_BROWSER` | Bake Chromium + Xvfb into the image at build time | -| `OPENCLAW_EXTRA_MOUNTS` | Extra host bind mounts (comma-separated `source:target[:opts]`) | -| `OPENCLAW_HOME_VOLUME` | Persist `/home/node` in a named Docker volume | -| `OPENCLAW_SANDBOX` | Opt in to sandbox bootstrap (`1`, `true`, `yes`, `on`) | -| `OPENCLAW_SKIP_ONBOARDING` | Skip the interactive onboarding step (`1`, `true`, `yes`, `on`) | -| `OPENCLAW_DOCKER_SOCKET` | Override the Docker socket path | -| `OPENCLAW_DISABLE_BONJOUR` | Force Bonjour/mDNS advertising on (`0`) or off (`1`); see [Bonjour / mDNS](#bonjour--mdns) | -| `OPENCLAW_DISABLE_BUNDLED_SOURCE_OVERLAYS` | Disable bundled plugin source bind-mount overlays | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | Shared OTLP/HTTP collector endpoint for OpenTelemetry export | -| `OTEL_EXPORTER_OTLP_*_ENDPOINT` | Signal-specific OTLP endpoints for traces, metrics, or logs | -| `OTEL_EXPORTER_OTLP_PROTOCOL` | OTLP protocol override. Only `http/protobuf` is supported today | -| `OTEL_SERVICE_NAME` | Service name used for OpenTelemetry resources | -| `OTEL_SEMCONV_STABILITY_OPT_IN` | Opt in to latest experimental GenAI semantic attributes | -| `OPENCLAW_OTEL_PRELOADED` | Skip starting a second OpenTelemetry SDK when one is preloaded | +| Variable | Purpose | +| ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `OPENCLAW_IMAGE` | Use a remote image instead of building locally | +| `OPENCLAW_IMAGE_APT_PACKAGES` | Install extra apt packages during build (space-separated). Legacy alias: `OPENCLAW_DOCKER_APT_PACKAGES` | +| `OPENCLAW_IMAGE_PIP_PACKAGES` | Install extra Python packages during build (space-separated) | +| `OPENCLAW_EXTENSIONS` | Compile/package supported selected plugins and install their runtime dependencies (comma- or space-separated ids) | +| `OPENCLAW_DOCKER_BUILD_NODE_OPTIONS` | Override the local source-build Node options (default `--max-old-space-size=8192`) | +| `OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB` | Override the local source-build tsdown heap in MB | +| `OPENCLAW_DOCKER_BUILD_SKIP_DTS` | Skip declaration output during runtime-only local image builds (default `1`) | +| `OPENCLAW_INSTALL_BROWSER` | Bake Chromium + Xvfb into the image at build time | +| `OPENCLAW_EXTRA_MOUNTS` | Extra host bind mounts (comma-separated `source:target[:opts]`) | +| `OPENCLAW_HOME_VOLUME` | Persist `/home/node` in a named Docker volume | +| `OPENCLAW_SANDBOX` | Opt in to sandbox bootstrap (`1`, `true`, `yes`, `on`) | +| `OPENCLAW_SKIP_ONBOARDING` | Skip the interactive onboarding step (`1`, `true`, `yes`, `on`) | +| `OPENCLAW_DOCKER_SOCKET` | Override the Docker socket path | +| `OPENCLAW_DISABLE_BONJOUR` | Force Bonjour/mDNS advertising on (`0`) or off (`1`); see [Bonjour / mDNS](#bonjour--mdns) | +| `OPENCLAW_DISABLE_BUNDLED_SOURCE_OVERLAYS` | Disable bundled plugin source bind-mount overlays | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Shared OTLP/HTTP collector endpoint for OpenTelemetry export | +| `OTEL_EXPORTER_OTLP_*_ENDPOINT` | Signal-specific OTLP endpoints for traces, metrics, or logs | +| `OTEL_EXPORTER_OTLP_PROTOCOL` | OTLP protocol override. Only `http/protobuf` is supported today | +| `OTEL_SERVICE_NAME` | Service name used for OpenTelemetry resources | +| `OTEL_SEMCONV_STABILITY_OPT_IN` | Opt in to latest experimental GenAI semantic attributes | +| `OPENCLAW_OTEL_PRELOADED` | Skip starting a second OpenTelemetry SDK when one is preloaded | The official image ships no Homebrew. During onboarding, OpenClaw hides brew-only skill dependency installers in a Linux container without `brew`; provide those dependencies through a custom image or install manually. Use `OPENCLAW_IMAGE_APT_PACKAGES` for Debian-packaged dependencies and `OPENCLAW_IMAGE_PIP_PACKAGES` for Python dependencies (runs `python3 -m pip install --break-system-packages` at build time, so pin versions and use only indexes you trust). @@ -183,6 +183,70 @@ If Docker reports `ResourceExhausted`, `cannot allocate memory`, or aborts durin OPENCLAW_DOCKER_BUILD_NODE_OPTIONS=--max-old-space-size=4096 OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB=4096 ``` +### Source-built images with selected plugins + +`OPENCLAW_EXTENSIONS` selects plugin manifest ids from the source checkout; +existing source-directory names are also accepted when they differ. The Docker +build resolves the selection to source directories once, installs production +dependencies, and, when a selected plugin is published separately with +`openclaw.build.bundledDist: false`, compiles its runtime into the root bundled +dist. This Docker-only packaging does not change the plugin's npm or ClawHub +artifact contract. Unknown, invalid, or ambiguous ids fail the image build. +Known dependency/source-only ids keep their existing source and dependency +staging without gaining a compiled root dist entry. A selected plugin with +unified build entries must compile successfully; unselected external plugin +source and runtime output are pruned. + +For example, these commands build separate, multi-architecture standalone +FakeCo gateway images for ClickClack, Slack, and Microsoft Teams. ClawRouter is +already part of the root OpenClaw runtime, so the ClickClack image selects only +`clickclack`. The explicit empty browser argument keeps the default image free +of Chromium: + +```bash +SOURCE_SHA="$(git rev-parse HEAD)" +BUILD_TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +REGISTRY="registry.example.com/fakeco" + +build_gateway_image() { + gateway="$1" + selected_plugin="$2" + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --build-arg "GIT_COMMIT=${SOURCE_SHA}" \ + --build-arg "OPENCLAW_BUILD_TIMESTAMP=${BUILD_TIMESTAMP}" \ + --build-arg "OPENCLAW_EXTENSIONS=${selected_plugin}" \ + --build-arg OPENCLAW_INSTALL_BROWSER= \ + --provenance=mode=max \ + --sbom=true \ + --tag "${REGISTRY}/openclaw-${gateway}:${SOURCE_SHA}" \ + --push \ + . +} + +build_gateway_image clickclack clickclack +build_gateway_image slack slack +build_gateway_image teams msteams +``` + +Use `--platform linux/arm64 --load` or `--platform linux/amd64 --load` for a +single native local build. Multi-platform output and attached SBOM/provenance +require a registry or another Buildx output that preserves attestations. After +pushing, inspect the manifest and deploy the immutable digest rather than the +mutable source-SHA tag: + +```bash +docker buildx imagetools inspect \ + "${REGISTRY}/openclaw-clickclack:${SOURCE_SHA}" +# Deploy: registry.example.com/fakeco/openclaw-clickclack@sha256: +``` + +These images are for standalone OCI-based gateways and generic Docker users. +Crabhelm-managed gateways do not consume them: that delivery path builds a +separate x86_64 appliance archive containing an OpenClaw npm tarball and pins +the Node, archive, and manifest digests. Build that appliance independently +from the same landed OpenClaw source. + To test bundled plugin source against a packaged image, mount one plugin source directory over its packaged source path, e.g. `OPENCLAW_EXTRA_MOUNTS=/path/to/fork/extensions/synology-chat:/app/extensions/synology-chat:ro`. That overrides the matching compiled `/app/dist/extensions/synology-chat` bundle for the same plugin id. ### Observability diff --git a/docs/install/podman.md b/docs/install/podman.md index 940288e8da08..65b0a422ea52 100644 --- a/docs/install/podman.md +++ b/docs/install/podman.md @@ -36,7 +36,7 @@ The model: | `OPENCLAW_IMAGE` / `OPENCLAW_PODMAN_IMAGE` | Use an existing/pulled image instead of building `openclaw:local` | | `OPENCLAW_IMAGE_APT_PACKAGES` | Install extra apt packages during image build (also accepts legacy `OPENCLAW_DOCKER_APT_PACKAGES`) | | `OPENCLAW_IMAGE_PIP_PACKAGES` | Install extra Python packages during image build; pin versions and use only package indexes you trust | - | `OPENCLAW_EXTENSIONS` | Pre-install plugin dependencies at build time | + | `OPENCLAW_EXTENSIONS` | Compile/package supported selected plugins and install their runtime dependencies | | `OPENCLAW_INSTALL_BROWSER` | Pre-install Chromium and Xvfb for browser automation (set to `1`) | For Quadlet-managed setup instead (Linux + systemd user services only): diff --git a/docs/providers/clawrouter.md b/docs/providers/clawrouter.md index 43f1a03fb20a..cf8924074896 100644 --- a/docs/providers/clawrouter.md +++ b/docs/providers/clawrouter.md @@ -125,6 +125,13 @@ credential, update the external Secret that supplies `CLAWROUTER_API_KEY` and restart the gateway workload so the new process environment is loaded. The config file and model reference do not change. +For a source-built standalone Docker gateway, ClawRouter is already included in +the root runtime. Select only the channel plugin that needs separate packaging, +such as `OPENCLAW_EXTENSIONS=clickclack`, `slack`, or `msteams`; see +[source-built images with selected plugins](/install/docker#source-built-images-with-selected-plugins). +Archive/appliance deployments must package the same landed source through their +own artifact pipeline rather than consuming the OCI image. + ## Readiness and live proof These checks prove different boundaries; do not substitute one for another: diff --git a/package.json b/package.json index c300c2267b53..c73a395e52f6 100644 --- a/package.json +++ b/package.json @@ -1861,6 +1861,7 @@ "test:docker:qr": "bash scripts/e2e/qr-import-docker.sh", "test:docker:rerun": "node scripts/docker-e2e-rerun.mjs", "test:docker:root-managed-vps-upgrade": "env OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE=1 OPENCLAW_UPGRADE_SURVIVOR_ROOT_MANAGED_VPS=1 OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC=${OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC:-openclaw@2026.5.7} OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT=${OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT:-1500s} bash scripts/e2e/upgrade-survivor-docker.sh", + "test:docker:selected-plugins": "bash scripts/e2e/docker-selected-plugins.sh", "test:docker:session-runtime-context": "bash scripts/e2e/session-runtime-context-docker.sh", "test:docker:skill-install": "bash scripts/e2e/skill-install-docker.sh", "test:docker:timings": "node scripts/docker-e2e-timings.mjs", diff --git a/scripts/check-docker-e2e-boundaries.mjs b/scripts/check-docker-e2e-boundaries.mjs index 4336c86633a2..4f52dbd51462 100644 --- a/scripts/check-docker-e2e-boundaries.mjs +++ b/scripts/check-docker-e2e-boundaries.mjs @@ -26,7 +26,10 @@ const livePackageBackedLanes = new Set([ ]); // These lanes intentionally build a focused source-checkout image instead of // consuming the shared package E2E images. -const sourceCheckoutImageLanes = new Set(["plugin-binding-command-escape"]); +const sourceCheckoutImageLanes = new Set([ + "docker-selected-plugins", + "plugin-binding-command-escape", +]); function readText(relativePath) { return fs.readFileSync(path.join(ROOT_DIR, relativePath), "utf8"); diff --git a/scripts/e2e/docker-selected-plugins.sh b/scripts/e2e/docker-selected-plugins.sh new file mode 100755 index 000000000000..7640c19e527e --- /dev/null +++ b/scripts/e2e/docker-selected-plugins.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-build.sh" +source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh" + +IMAGE_NAME="${OPENCLAW_DOCKER_SELECTED_PLUGINS_E2E_IMAGE:-openclaw-docker-selected-plugins-e2e:local}" +DEPENDENCY_ONLY_IMAGE="${IMAGE_NAME}-dependency-only" +CONTAINER_NAME="openclaw-docker-selected-plugins-e2e-$$" +SELECTED_PLUGINS="${OPENCLAW_DOCKER_SELECTED_PLUGINS:-slack,msteams clickclack,slack}" +BUILD_GIT_COMMIT="${OPENCLAW_DOCKER_SELECTED_PLUGINS_E2E_GIT_COMMIT:-0123456789abcdef0123456789abcdef01234567}" +BUILD_TIMESTAMP="${OPENCLAW_DOCKER_SELECTED_PLUGINS_E2E_BUILD_TIMESTAMP:-2026-07-10T12:34:56.000Z}" +UNKNOWN_LOG="$(mktemp -t openclaw-docker-selected-plugins-unknown.XXXXXX)" +RUN_LOG="$(mktemp -t openclaw-docker-selected-plugins-run.XXXXXX)" +DOCKER_COMMAND_TIMEOUT="${OPENCLAW_DOCKER_SELECTED_PLUGINS_RUN_TIMEOUT:-900s}" +DEPENDENCY_ONLY_IMAGE_BUILT=0 + +cleanup() { + docker_e2e_docker_cmd rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + if [ "$DEPENDENCY_ONLY_IMAGE_BUILT" = "1" ]; then + docker_e2e_docker_cmd image rm -f "$DEPENDENCY_ONLY_IMAGE" >/dev/null 2>&1 || true + fi + rm -f "$UNKNOWN_LOG" "$RUN_LOG" +} +trap cleanup EXIT + +if [ "${OPENCLAW_SKIP_DOCKER_BUILD:-0}" = "1" ]; then + echo "Reusing selected-plugin image: $IMAGE_NAME" + docker_e2e_docker_cmd image inspect "$IMAGE_NAME" >/dev/null +else + echo "Proving unknown selected plugins fail closed..." + set +e + docker_e2e_timeout_cmd "${OPENCLAW_DOCKER_SELECTED_PLUGINS_BUILD_TIMEOUT:-3600s}" \ + env DOCKER_BUILDKIT=1 docker build \ + --target workspace-deps \ + --build-arg OPENCLAW_EXTENSIONS=missing-plugin \ + -f "$ROOT_DIR/Dockerfile" \ + "$ROOT_DIR" >"$UNKNOWN_LOG" 2>&1 + unknown_status=$? + set -e + if [ "$unknown_status" -eq 0 ] || ! grep -Fq \ + "unknown OPENCLAW_EXTENSIONS plugin id: missing-plugin" "$UNKNOWN_LOG"; then + echo "Unknown selected-plugin build did not fail closed as expected" >&2 + docker_e2e_print_log "$UNKNOWN_LOG" + exit 1 + fi + + echo "Proving manifest ids and known dependency-only plugins remain stageable..." + docker_build_run docker-selected-plugins-dependency-only \ + --target workspace-deps \ + --build-arg OPENCLAW_EXTENSIONS=whatsapp,qqbot,kimi \ + -t "$DEPENDENCY_ONLY_IMAGE" \ + -f "$ROOT_DIR/Dockerfile" \ + "$ROOT_DIR" + DEPENDENCY_ONLY_IMAGE_BUILT=1 + docker_e2e_docker_run_cmd run --rm \ + --entrypoint sh \ + "$DEPENDENCY_ONLY_IMAGE" \ + -c 'test -f /out/extensions/whatsapp/package.json && test -f /out/extensions/qqbot/package.json && test -f /out/extensions/kimi-coding/package.json && grep -qx kimi-coding /out/openclaw-selected-plugin-dirs' + + echo "Building selected-plugin runtime image: $IMAGE_NAME" + docker_build_run docker-selected-plugins-build \ + --build-arg "GIT_COMMIT=$BUILD_GIT_COMMIT" \ + --build-arg "OPENCLAW_BUILD_TIMESTAMP=$BUILD_TIMESTAMP" \ + --build-arg "OPENCLAW_EXTENSIONS=$SELECTED_PLUGINS" \ + -t "$IMAGE_NAME" \ + -f "$ROOT_DIR/Dockerfile" \ + "$ROOT_DIR" +fi + +echo "Inspecting selected plugins from the final runtime image..." +if ! docker_e2e_docker_run_cmd run --rm \ + --name "$CONTAINER_NAME" \ + --entrypoint bash \ + -e "OPENCLAW_E2E_EXPECTED_GIT_COMMIT=$BUILD_GIT_COMMIT" \ + -e "OPENCLAW_E2E_EXPECTED_BUILD_TIMESTAMP=$BUILD_TIMESTAMP" \ + -v "$ROOT_DIR/scripts/e2e/lib/docker-selected-plugins:/openclaw-e2e:ro" \ + "$IMAGE_NAME" \ + /openclaw-e2e/scenario.sh >"$RUN_LOG" 2>&1; then + echo "Selected-plugin Docker E2E failed" >&2 + docker_e2e_print_log "$RUN_LOG" + exit 1 +fi + +docker_e2e_print_log "$RUN_LOG" +echo "Selected-plugin Docker E2E passed" diff --git a/scripts/e2e/lib/docker-selected-plugins/assertions.mjs b/scripts/e2e/lib/docker-selected-plugins/assertions.mjs new file mode 100644 index 000000000000..b823cbc40c6c --- /dev/null +++ b/scripts/e2e/lib/docker-selected-plugins/assertions.mjs @@ -0,0 +1,118 @@ +import fs from "node:fs"; +import path from "node:path"; + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function assertFile(filePath) { + assert(fs.statSync(filePath).isFile(), `expected file: ${filePath}`); +} + +function assertAbsent(filePath) { + assert(!fs.existsSync(filePath), `expected path to be absent: ${filePath}`); +} + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +const selected = { + clickclack: { + entries: ["index.js"], + capability: "channel", + }, + slack: { + entries: ["index.js", "setup-entry.js"], + capability: "channel", + }, + msteams: { + entries: ["index.js", "setup-entry.js"], + capability: "channel", + }, + clawrouter: { + entries: ["index.js"], + capability: "text-inference", + }, +}; + +const buildInfo = readJson("/app/dist/build-info.json"); +const expectedCommit = process.env.OPENCLAW_E2E_EXPECTED_GIT_COMMIT?.toLowerCase(); +const expectedBuiltAt = new Date( + process.env.OPENCLAW_E2E_EXPECTED_BUILD_TIMESTAMP ?? "", +).toISOString(); +assert(buildInfo.commit === expectedCommit, `unexpected build commit: ${buildInfo.commit}`); +assert(buildInfo.builtAt === expectedBuiltAt, `unexpected build timestamp: ${buildInfo.builtAt}`); + +for (const [pluginId, expected] of Object.entries(selected)) { + const pluginRoot = path.join("/app/dist/extensions", pluginId); + for (const entry of expected.entries) { + assertFile(path.join(pluginRoot, entry)); + } + assertFile(path.join(pluginRoot, "openclaw.plugin.json")); + assertFile(path.join(pluginRoot, "package.json")); + + const manifest = readJson(path.join(pluginRoot, "openclaw.plugin.json")); + const packageJson = readJson(path.join(pluginRoot, "package.json")); + assert(manifest.id === pluginId, `unexpected ${pluginId} manifest id: ${manifest.id}`); + assert( + packageJson.openclaw?.extensions?.includes("./index.js"), + `${pluginId} package entry was not rewritten to ./index.js`, + ); + if (expected.entries.includes("setup-entry.js")) { + assert( + packageJson.openclaw?.setupEntry === "./setup-entry.js", + `${pluginId} setup entry was not rewritten to ./setup-entry.js`, + ); + } + + const inspect = readJson(`/tmp/openclaw-${pluginId}-inspect.json`); + assert(inspect.plugin?.id === pluginId, `unexpected ${pluginId} inspect id`); + assert(inspect.plugin?.status === "loaded", `${pluginId} runtime did not load`); + assert(inspect.plugin?.origin === "bundled", `${pluginId} did not load from bundled dist`); + assert( + inspect.capabilities?.some( + (entry) => entry?.kind === expected.capability && entry.ids?.includes(pluginId), + ), + `${pluginId} did not register ${expected.capability} capability`, + ); +} + +for (const pluginId of ["clickclack", "slack"]) { + const packageJson = readJson(`/app/dist/extensions/${pluginId}/package.json`); + assert( + packageJson.openclaw?.build?.bundledDist === false, + `${pluginId} bundledDist release metadata changed`, + ); +} + +const declaredDependencies = { + clickclack: ["ws"], + slack: ["@slack/bolt", "@slack/web-api"], + msteams: ["@microsoft/teams.apps"], +}; +for (const [pluginId, dependencies] of Object.entries(declaredDependencies)) { + const packageJson = readJson(`/app/dist/extensions/${pluginId}/package.json`); + for (const dependency of dependencies) { + assert( + typeof packageJson.dependencies?.[dependency] === "string", + `${pluginId} package metadata omitted ${dependency}`, + ); + } +} + +for (const dependency of ["@microsoft/teams.apps", "@slack/bolt", "@slack/web-api", "ws"]) { + assertFile(path.join("/app/node_modules", dependency, "package.json")); +} + +assertFile("/app/dist/extensions/slack/skills/slack/SKILL.md"); +assertAbsent("/app/dist/extensions/amazon-bedrock"); +assertAbsent("/app/extensions/amazon-bedrock"); +assertAbsent("/app/node_modules/@aws-sdk/client-bedrock"); +assertAbsent("/app/dist/extensions/signal"); +assertAbsent("/app/extensions/signal"); +assertAbsent("/home/node/.cache/ms-playwright"); + +console.log(`Selected-plugin runtime proof passed (${process.arch})`); diff --git a/scripts/e2e/lib/docker-selected-plugins/scenario.sh b/scripts/e2e/lib/docker-selected-plugins/scenario.sh new file mode 100755 index 000000000000..e7138492af95 --- /dev/null +++ b/scripts/e2e/lib/docker-selected-plugins/scenario.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +export HOME=/tmp/openclaw-docker-selected-plugins +export OPENCLAW_STATE_DIR="$HOME/.openclaw" +export OPENCLAW_CONFIG_PATH="$OPENCLAW_STATE_DIR/openclaw.json" +export OPENCLAW_DISABLE_BUNDLED_SOURCE_OVERLAYS=1 + +mkdir -p "$OPENCLAW_STATE_DIR" +node --input-type=module <<'NODE' +import fs from "node:fs"; + +const entries = Object.fromEntries( + ["clickclack", "slack", "msteams"].map((id) => [id, { enabled: true }]), +); +fs.writeFileSync( + process.env.OPENCLAW_CONFIG_PATH, + `${JSON.stringify({ plugins: { entries } }, null, 2)}\n`, + { mode: 0o600 }, +); +NODE + +for plugin_id in clickclack slack msteams clawrouter; do + node /app/openclaw.mjs plugins inspect "$plugin_id" --runtime --json \ + >"/tmp/openclaw-${plugin_id}-inspect.json" +done + +node /openclaw-e2e/assertions.mjs diff --git a/scripts/lib/bundled-plugin-build-entries-types.d.ts b/scripts/lib/bundled-plugin-build-entries-types.d.ts index 27930b13c80b..188f0c3abad5 100644 --- a/scripts/lib/bundled-plugin-build-entries-types.d.ts +++ b/scripts/lib/bundled-plugin-build-entries-types.d.ts @@ -13,6 +13,7 @@ export type BundledPluginBuildEntryParams = { }; export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS: Set; +export const DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV: string; export function collectRootPackageExcludedExtensionDirs( params?: BundledPluginBuildEntryParams, ): Set; diff --git a/scripts/lib/bundled-plugin-build-entries.mjs b/scripts/lib/bundled-plugin-build-entries.mjs index 69cb54510195..7d6d3850ab10 100644 --- a/scripts/lib/bundled-plugin-build-entries.mjs +++ b/scripts/lib/bundled-plugin-build-entries.mjs @@ -14,6 +14,8 @@ const TOP_LEVEL_PUBLIC_SURFACE_EXTENSIONS = new Set([".ts", ".js", ".mts", ".cts export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS = new Set(["qa-channel", "qa-lab", "qa-matrix"]); const EXCLUDED_CORE_BUNDLED_PLUGIN_DIRS = new Set(["qqbot", "whatsapp"]); const BUNDLED_PLUGIN_BUILD_IDS_ENV = "OPENCLAW_BUNDLED_PLUGIN_BUILD_IDS"; +export const DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV = "OPENCLAW_INTERNAL_DOCKER_BUILD_PLUGIN_IDS"; +const PLUGIN_ID_RE = /^[a-z0-9][a-z0-9-]*$/u; const TOP_LEVEL_PRIVATE_TEST_SURFACE_RE = /(?:^|[._-])(?:test|spec|test-support|test-helpers|test-fixtures|test-harness|mock-setup)(?:[._-]|$)/u; const toPosixPath = (value) => value.replaceAll("\\", "/"); @@ -31,6 +33,28 @@ function parseBundledPluginBuildIdFilter(env = process.env) { ); } +function parseDockerSelectedPluginBuildIdFilter(env = process.env) { + const raw = env[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]; + if (typeof raw !== "string" || raw.trim() === "") { + return null; + } + const ids = new Set( + raw + .split(/[\s,]+/u) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0), + ); + const invalidIds = [...ids].filter((id) => !PLUGIN_ID_RE.test(id)); + if (invalidIds.length > 0) { + throw new Error( + `${DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV} contains invalid plugin id(s): ${invalidIds + .toSorted((left, right) => left.localeCompare(right)) + .join(", ")}`, + ); + } + return ids; +} + function readBundledPluginPackageJson(packageJsonPath, options = {}) { if (!(options.hasPackageJson ?? fs.existsSync(packageJsonPath))) { return null; @@ -184,7 +208,8 @@ function collectBundledPluginCandidates(cwd, extensionsRoot) { relativeFiles: null, topLevelPublicSurfaceEntries: collectTopLevelPublicSurfaceEntries(pluginDir), }; - }); + }) + .toSorted((left, right) => left.dirName.localeCompare(right.dirName)); } /** Collect all bundled plugin build entries for the current checkout. */ @@ -192,9 +217,11 @@ export function collectBundledPluginBuildEntries(params = {}) { const cwd = params.cwd ?? process.cwd(); const env = params.env ?? process.env; const extensionsRoot = path.join(cwd, BUNDLED_PLUGIN_ROOT_DIR); + const dockerSelectedBuildIds = parseDockerSelectedPluginBuildIdFilter(env); + const candidates = collectBundledPluginCandidates(cwd, extensionsRoot); const entries = []; - for (const candidate of collectBundledPluginCandidates(cwd, extensionsRoot)) { + for (const candidate of candidates) { const { dirName, pluginDir, relativeFiles, topLevelPublicSurfaceEntries } = candidate; const manifestPath = path.join(pluginDir, "openclaw.plugin.json"); const hasManifest = @@ -216,7 +243,7 @@ export function collectBundledPluginBuildEntries(params = {}) { if (!shouldBuildBundledCluster(dirName, env, { packageJson })) { continue; } - if (!shouldBuildBundledDistEntry(packageJson)) { + if (!shouldBuildBundledDistEntry(packageJson) && !dockerSelectedBuildIds?.has(dirName)) { continue; } if (EXCLUDED_CORE_BUNDLED_PLUGIN_DIRS.has(dirName)) { @@ -237,6 +264,18 @@ export function collectBundledPluginBuildEntries(params = {}) { }); } + if (dockerSelectedBuildIds) { + const knownIds = new Set(candidates.map((candidate) => candidate.dirName)); + const unknownIds = [...dockerSelectedBuildIds].filter((id) => !knownIds.has(id)); + if (unknownIds.length > 0) { + throw new Error( + `${DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV} references unknown plugin id(s): ${unknownIds + .toSorted((left, right) => left.localeCompare(right)) + .join(", ")}`, + ); + } + } + const filteredBuildIds = parseBundledPluginBuildIdFilter(env); if (!filteredBuildIds) { return entries; diff --git a/scripts/lib/docker-e2e-scenarios.mjs b/scripts/lib/docker-e2e-scenarios.mjs index 08ebd267f6ff..00f5eaf91b48 100644 --- a/scripts/lib/docker-e2e-scenarios.mjs +++ b/scripts/lib/docker-e2e-scenarios.mjs @@ -262,15 +262,22 @@ function kitchenSinkRpcLane() { } export const mainLanes = [ - serviceLane( - "compose-setup", - "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:compose-setup", + lane( + "docker-selected-plugins", + "OPENCLAW_SKIP_DOCKER_BUILD=0 pnpm test:docker:selected-plugins", { - stateScenario: "empty", - timeoutMs: 20 * 60 * 1000, - weight: 3, + e2eImageKind: false, + estimateSeconds: 600, + resources: ["docker"], + timeoutMs: 30 * 60 * 1000, + weight: 4, }, ), + serviceLane("compose-setup", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:compose-setup", { + stateScenario: "empty", + timeoutMs: 20 * 60 * 1000, + weight: 3, + }), npmLane( "docker-package-install", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:package-install", @@ -933,11 +940,10 @@ export function releasePathChunkLanes(chunk, options = {}) { export function allReleasePathLanes(options = {}) { const releaseProfile = normalizeReleaseProfile(options.releaseProfile); - return Object.keys(primaryReleasePathChunks) - .flatMap((chunk) => - releasePathChunkLanes(chunk, { - includeOpenWebUI: options.includeOpenWebUI, - releaseProfile, - }), - ); + return Object.keys(primaryReleasePathChunks).flatMap((chunk) => + releasePathChunkLanes(chunk, { + includeOpenWebUI: options.includeOpenWebUI, + releaseProfile, + }), + ); } diff --git a/scripts/lib/docker-plugin-selection.mjs b/scripts/lib/docker-plugin-selection.mjs new file mode 100644 index 000000000000..4a419c2dda63 --- /dev/null +++ b/scripts/lib/docker-plugin-selection.mjs @@ -0,0 +1,83 @@ +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const PLUGIN_ID_RE = /^[a-z0-9][a-z0-9-]*$/u; + +function readManifestId(pluginDir) { + const manifestPath = path.join(pluginDir, "openclaw.plugin.json"); + if (!fs.existsSync(manifestPath)) { + return null; + } + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + return typeof manifest.id === "string" && manifest.id.length > 0 ? manifest.id : null; +} + +function collectPluginIdentities(extensionsRoot) { + return fs + .readdirSync(extensionsRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => { + const pluginDir = path.join(extensionsRoot, entry.name); + const hasPackageJson = fs.existsSync(path.join(pluginDir, "package.json")); + const manifestId = readManifestId(pluginDir); + return { + dirName: entry.name, + manifestId, + known: hasPackageJson || manifestId !== null, + }; + }) + .filter((entry) => entry.known) + .toSorted((left, right) => left.dirName.localeCompare(right.dirName)); +} + +/** Resolve public Docker selections to the source directories used by build and prune steps. */ +export function resolveDockerPluginSelection(params) { + const selection = typeof params.selection === "string" ? params.selection : ""; + const selectedIds = new Set( + selection + .split(/[\s,]+/u) + .map((entry) => entry.trim()) + .filter(Boolean), + ); + const plugins = collectPluginIdentities(params.extensionsRoot); + const resolvedDirs = new Set(); + + for (const selectedId of selectedIds) { + if (!PLUGIN_ID_RE.test(selectedId)) { + throw new Error(`invalid OPENCLAW_EXTENSIONS plugin id: ${selectedId}`); + } + const matches = plugins.filter( + (plugin) => plugin.dirName === selectedId || plugin.manifestId === selectedId, + ); + if (matches.length === 0) { + throw new Error(`unknown OPENCLAW_EXTENSIONS plugin id: ${selectedId}`); + } + if (matches.length > 1) { + throw new Error( + `ambiguous OPENCLAW_EXTENSIONS plugin id: ${selectedId} (${matches + .map((plugin) => plugin.dirName) + .join(", ")})`, + ); + } + resolvedDirs.add(matches[0].dirName); + } + + return [...resolvedDirs].toSorted((left, right) => left.localeCompare(right)); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + const resolved = resolveDockerPluginSelection({ + extensionsRoot: process.argv[2], + selection: process.argv[3] ?? "", + }); + if (resolved.length > 0) { + process.stdout.write(`${resolved.join("\n")}\n`); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`ERROR: ${message}`); + process.exitCode = 1; + } +} diff --git a/scripts/package-openclaw-for-docker.mjs b/scripts/package-openclaw-for-docker.mjs index 683fb8188aed..333d77a04af3 100644 --- a/scripts/package-openclaw-for-docker.mjs +++ b/scripts/package-openclaw-for-docker.mjs @@ -7,6 +7,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import * as tar from "tar"; +import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "./lib/bundled-plugin-build-entries.mjs"; import { preparePackageChangelog, restorePackageChangelog } from "./package-changelog.mjs"; const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -22,6 +23,11 @@ const MAX_TIMER_TIMEOUT_MS = 2_147_000_000; const AI_RUNTIME_PACKAGE = "@openclaw/ai"; const AI_RUNTIME_BACKUP_DIR = ".openclaw-ai-package-backup"; const ACTIVE_CHILD_KILLERS = new Set(); +const PACKAGE_BUILD_PLUGIN_SELECTION_ENV_NAMES = [ + "OPENCLAW_EXTENSIONS", + "OPENCLAW_DOCKER_BUILD_EXTENSIONS", + DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV, +]; const SIGNAL_EXIT_CODES = { SIGHUP: 129, SIGINT: 130, @@ -365,13 +371,19 @@ const PACKAGE_ARTIFACT_BUILD_STEPS = [ export async function buildPackageArtifacts(sourceDir, options = {}) { const runImpl = options.runImpl ?? run; + const buildEnv = { + ...process.env, + OPENCLAW_BUILD_ALL_NO_PNPM: "1", + OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "0", + }; + for (const envName of PACKAGE_BUILD_PLUGIN_SELECTION_ENV_NAMES) { + delete buildEnv[envName]; + } for (const step of PACKAGE_ARTIFACT_BUILD_STEPS) { console.error(`==> ${step.label}`); await runImpl(step.command, step.args, sourceDir, { env: { - ...process.env, - OPENCLAW_BUILD_ALL_NO_PNPM: "1", - OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "0", + ...buildEnv, }, timeoutMs: resolveTimeoutMs( "OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS", diff --git a/scripts/release-check.ts b/scripts/release-check.ts index a98b5326baf5..a1ed2153be98 100755 --- a/scripts/release-check.ts +++ b/scripts/release-check.ts @@ -73,6 +73,9 @@ type ReleaseCheckCommandInvocation = { }; const rootPackageExcludedExtensionDirs = collectRootPackageExcludedExtensionDirs(); +const rootPackageExcludedExtensionPrefixes = [...rootPackageExcludedExtensionDirs].map( + (extensionId) => `dist/extensions/${extensionId}/`, +); const requiredPathGroups = [ "npm-shrinkwrap.json", PACKAGE_DIST_INVENTORY_RELATIVE_PATH, @@ -105,6 +108,7 @@ const requiredPathGroups = [ "dist/control-ui/index.html", ]; const forbiddenPrefixes = [ + ...rootPackageExcludedExtensionPrefixes, ...LOCAL_BUILD_METADATA_DIST_PATHS, "dist-runtime/", "dist/OpenClaw.app/", diff --git a/src/dockerfile.test.ts b/src/dockerfile.test.ts index 7968e07440d1..678ebe0f7898 100644 --- a/src/dockerfile.test.ts +++ b/src/dockerfile.test.ts @@ -152,7 +152,15 @@ describe("Dockerfile", () => { expect(extensionManifestIndex).toBeGreaterThan(-1); expect(dockerfile).toContain("for manifest in /tmp/packages/*/package.json"); expect(dockerfile).toContain( - `if [ -f "/tmp/\${OPENCLAW_BUNDLED_PLUGIN_DIR}/$ext/package.json" ]; then`, + "--mount=type=bind,source=scripts/lib/docker-plugin-selection.mjs,target=/tmp/docker-plugin-selection.mjs,readonly", + ); + expect(dockerfile).toContain( + 'node /tmp/docker-plugin-selection.mjs "/tmp/${OPENCLAW_BUNDLED_PLUGIN_DIR}" "$OPENCLAW_EXTENSIONS"', + ); + expect(dockerfile).toContain("done < /out/openclaw-selected-plugin-dirs"); + expect(dockerfile).toContain(`if [ -f "$ext_dir/package.json" ]; then`); + expect(dockerfile).toContain( + "COPY --from=workspace-deps /out/openclaw-selected-plugin-dirs /tmp/openclaw-selected-plugin-dirs", ); expect(postinstallIndex).toBeLessThan(installIndex); expect(prepareIndex).toBeLessThan(installIndex); @@ -161,6 +169,24 @@ describe("Dockerfile", () => { expect(extensionManifestIndex).toBeLessThan(installIndex); }); + it("keeps validated plugin selection outside the build-context copy destination", async () => { + const dockerfile = await readFile(dockerfilePath, "utf8"); + const selectionCopyIndex = dockerfile.indexOf( + "COPY --from=workspace-deps /out/openclaw-selected-plugin-dirs /tmp/openclaw-selected-plugin-dirs", + ); + const buildContextCopyIndex = dockerfile.indexOf("COPY . ."); + + expect(selectionCopyIndex).toBeGreaterThan(-1); + expect(buildContextCopyIndex).toBeGreaterThan(selectionCopyIndex); + expect(dockerfile).not.toContain("/app/.openclaw-selected-plugin-dirs"); + expect(dockerfile).not.toContain("./.openclaw-selected-plugin-dirs"); + expect(dockerfile).toContain("grep -qx 'matrix' /tmp/openclaw-selected-plugin-dirs"); + expect(dockerfile).toContain( + 'selected_plugin_dirs="$(cat /tmp/openclaw-selected-plugin-dirs)"', + ); + expect(dockerfile).toContain('OPENCLAW_EXTENSIONS="$(cat /tmp/openclaw-selected-plugin-dirs)"'); + }); + it("copies root package lifecycle scripts before pnpm install", async () => { const [dockerfile, packageJsonText] = await Promise.all([ readFile(dockerfilePath, "utf8"), @@ -199,7 +225,7 @@ describe("Dockerfile", () => { "export OPENCLAW_BUILD_PRIVATE_QA=1 OPENCLAW_ENABLE_PRIVATE_QA_CLI=1", ); const buildDockerIndex = collapsed.indexOf( - 'OPENCLAW_RUN_NODE_SKIP_DTS_BUILD="$OPENCLAW_DOCKER_BUILD_SKIP_DTS" OPENCLAW_TSDOWN_MAX_OLD_SPACE_MB="$OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB" NODE_OPTIONS="$OPENCLAW_DOCKER_BUILD_NODE_OPTIONS" pnpm_config_verify_deps_before_run=false pnpm build:docker', + 'OPENCLAW_INTERNAL_DOCKER_BUILD_PLUGIN_IDS="$selected_plugin_dirs" OPENCLAW_RUN_NODE_SKIP_DTS_BUILD="$OPENCLAW_DOCKER_BUILD_SKIP_DTS" OPENCLAW_TSDOWN_MAX_OLD_SPACE_MB="$OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB" NODE_OPTIONS="$OPENCLAW_DOCKER_BUILD_NODE_OPTIONS" pnpm_config_verify_deps_before_run=false pnpm build:docker', ); const qaLabBuildIndex = collapsed.indexOf( "pnpm_config_verify_deps_before_run=false pnpm qa:lab:build", @@ -211,6 +237,9 @@ describe("Dockerfile", () => { expect(qaLabExtensionCheckIndex).toBeGreaterThan(-1); expect(buildDockerIndex).toBeGreaterThan(-1); + expect(collapsed).not.toContain( + 'OPENCLAW_DOCKER_BUILD_EXTENSIONS="$OPENCLAW_EXTENSIONS" OPENCLAW_RUN_NODE_SKIP_DTS_BUILD=', + ); expect(qaLabBuildIndex).toBeGreaterThan(-1); expect(qaLabDistCopyIndex).toBeGreaterThan(-1); expect(runtimeAssetsIndex).toBeGreaterThan(-1); @@ -246,12 +275,23 @@ describe("Dockerfile", () => { it("documents provenance arguments for manual source builds", async () => { const docs = await readFile(dockerInstallDocsPath, "utf8"); + const selectedPluginStart = docs.indexOf("### Source-built images with selected plugins"); + const selectedPluginEnd = docs.indexOf("### Observability", selectedPluginStart); + const selectedPluginDocs = docs.slice(selectedPluginStart, selectedPluginEnd); expect(docs).toContain('BUILD_GIT_COMMIT="$(git rev-parse HEAD)"'); expect(docs).toContain('BUILD_TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"'); expect(docs).toContain('--build-arg "GIT_COMMIT=${BUILD_GIT_COMMIT}"'); expect(docs).toContain('--build-arg "OPENCLAW_BUILD_TIMESTAMP=${BUILD_TIMESTAMP}"'); expect(docs).toContain("The Docker context excludes `.git`."); + expect(selectedPluginStart).toBeGreaterThan(-1); + expect(selectedPluginEnd).toBeGreaterThan(selectedPluginStart); + expect(selectedPluginDocs).toContain('SOURCE_SHA="$(git rev-parse HEAD)"'); + expect(selectedPluginDocs).toContain('BUILD_TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"'); + expect(selectedPluginDocs).toContain('--build-arg "GIT_COMMIT=${SOURCE_SHA}"'); + expect(selectedPluginDocs).toContain( + '--build-arg "OPENCLAW_BUILD_TIMESTAMP=${BUILD_TIMESTAMP}"', + ); }); it("prunes runtime dependencies and omitted plugin packages after the build stage", async () => { @@ -265,7 +305,7 @@ describe("Dockerfile", () => { expect(dockerfile).toContain("ARG OPENCLAW_DOCKER_BUILD_SKIP_DTS=1"); expect(dockerfile).toContain("ARG OPENCLAW_BUNDLED_PLUGIN_DIR"); expect(dockerfile).toContain( - "Opt-in plugin dependencies at build time (space- or comma-separated directory names).", + "Opt-in plugin dependencies and supported runtime builds (space- or comma-separated ids).", ); expect(dockerfile).toContain( 'Example: docker build --build-arg OPENCLAW_EXTENSIONS="diagnostics-otel,matrix" .', @@ -278,14 +318,14 @@ describe("Dockerfile", () => { "COPY --from=workspace-deps /out/${OPENCLAW_BUNDLED_PLUGIN_DIR}/ ./${OPENCLAW_BUNDLED_PLUGIN_DIR}/", ); expect(dockerfile).toContain( - 'OPENCLAW_EXTENSIONS="$OPENCLAW_EXTENSIONS" OPENCLAW_BUNDLED_PLUGIN_DIR="$OPENCLAW_BUNDLED_PLUGIN_DIR" node scripts/prune-docker-plugin-dist.mjs', + 'OPENCLAW_EXTENSIONS="$(cat /tmp/openclaw-selected-plugin-dirs)" OPENCLAW_BUNDLED_PLUGIN_DIR="$OPENCLAW_BUNDLED_PLUGIN_DIR" node scripts/prune-docker-plugin-dist.mjs', ); expect(dockerfile).toContain("readlink -f /app/node_modules/@openclaw/ai"); expect(dockerfile).toContain('mv "$ai_runtime_tmp/ai" /app/node_modules/@openclaw/ai'); expect(dockerfile).toContain("CI=true pnpm prune --prod \\"); expect(dockerfile.indexOf("CI=true pnpm prune --prod \\")).toBeLessThan( dockerfile.indexOf( - 'OPENCLAW_EXTENSIONS="$OPENCLAW_EXTENSIONS" OPENCLAW_BUNDLED_PLUGIN_DIR="$OPENCLAW_BUNDLED_PLUGIN_DIR" node scripts/prune-docker-plugin-dist.mjs', + 'OPENCLAW_EXTENSIONS="$(cat /tmp/openclaw-selected-plugin-dirs)" OPENCLAW_BUNDLED_PLUGIN_DIR="$OPENCLAW_BUNDLED_PLUGIN_DIR" node scripts/prune-docker-plugin-dist.mjs', ), ); expect(dockerfile).toContain("--config.offline=true"); diff --git a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts index 362de8ce4f59..0b57950ea7c2 100644 --- a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; import { describe, expect, it, vi } from "vitest"; +import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "../../../../scripts/lib/bundled-plugin-build-entries.mjs"; import { buildPackageArtifacts, packOpenClawPackageForDocker, @@ -157,13 +158,22 @@ describe("package-openclaw-for-docker", () => { args: string[]; cwd: string; noPnpm: string | undefined; + packageExtensions: string | undefined; + dockerBuildExtensions: string | undefined; + internalDockerBuildPluginIds: string | undefined; skipDts: string | undefined; timeoutMs: number | undefined; }> = []; const previousTimeout = process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS; const previousSkipDts = process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD; + const previousPackageExtensions = process.env.OPENCLAW_EXTENSIONS; + const previousDockerBuildExtensions = process.env.OPENCLAW_DOCKER_BUILD_EXTENSIONS; + const previousInternalPluginIds = process.env[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]; process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = "1234"; process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD = "1"; + process.env.OPENCLAW_EXTENSIONS = "clickclack"; + process.env.OPENCLAW_DOCKER_BUILD_EXTENSIONS = "slack"; + process.env[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV] = "msteams"; try { await buildPackageArtifacts("/repo", { @@ -178,6 +188,9 @@ describe("package-openclaw-for-docker", () => { args, cwd, noPnpm: options.env?.OPENCLAW_BUILD_ALL_NO_PNPM, + packageExtensions: options.env?.OPENCLAW_EXTENSIONS, + dockerBuildExtensions: options.env?.OPENCLAW_DOCKER_BUILD_EXTENSIONS, + internalDockerBuildPluginIds: options.env?.[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV], skipDts: options.env?.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD, timeoutMs: options.timeoutMs, }); @@ -194,6 +207,17 @@ describe("package-openclaw-for-docker", () => { } else { process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD = previousSkipDts; } + for (const [envName, previousValue] of [ + ["OPENCLAW_EXTENSIONS", previousPackageExtensions], + ["OPENCLAW_DOCKER_BUILD_EXTENSIONS", previousDockerBuildExtensions], + [DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV, previousInternalPluginIds], + ] as const) { + if (previousValue === undefined) { + delete process.env[envName]; + } else { + process.env[envName] = previousValue; + } + } } expect(calls).toEqual([ @@ -201,7 +225,10 @@ describe("package-openclaw-for-docker", () => { command: "node", args: ["scripts/build-all.mjs", "ciArtifacts"], cwd: "/repo", + dockerBuildExtensions: undefined, + internalDockerBuildPluginIds: undefined, noPnpm: "1", + packageExtensions: undefined, skipDts: "0", timeoutMs: 1234, }, diff --git a/test/release-check.test.ts b/test/release-check.test.ts index 315e33da2730..512aac6e4c9c 100644 --- a/test/release-check.test.ts +++ b/test/release-check.test.ts @@ -525,6 +525,21 @@ describe("collectForbiddenPackPaths", () => { ).toEqual([...LOCAL_BUILD_METADATA_DIST_PATHS]); }); + it("blocks Docker-selected external plugin trees from the core npm pack", () => { + expect( + collectForbiddenPackPaths([ + "dist/index.js", + "dist/extensions/clickclack/index.js", + "dist/extensions/slack/setup-entry.js", + "dist/extensions/msteams/openclaw.plugin.json", + ]), + ).toEqual([ + "dist/extensions/clickclack/index.js", + "dist/extensions/msteams/openclaw.plugin.json", + "dist/extensions/slack/setup-entry.js", + ]); + }); + it("keeps local build metadata excluded by package files", () => { const pkg = JSON.parse(readFileSync("package.json", "utf8")) as { files?: string[] }; for (const entry of LOCAL_BUILD_METADATA_DIST_PATHS) { diff --git a/test/scripts/bundled-plugin-build-entries.test.ts b/test/scripts/bundled-plugin-build-entries.test.ts index 2de724277a6d..212035e98291 100644 --- a/test/scripts/bundled-plugin-build-entries.test.ts +++ b/test/scripts/bundled-plugin-build-entries.test.ts @@ -1,13 +1,17 @@ // Bundled Plugin Build Entries tests cover bundled plugin build entries script behavior. import fs from "node:fs"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { collectRootPackageExcludedExtensionDirs, + DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV, listBundledPluginBuildEntries, listBundledPluginPackArtifacts, } from "../../scripts/lib/bundled-plugin-build-entries.mjs"; import { expectNoNodeFsScans } from "../../src/test-utils/fs-scan-assertions.js"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); function expectNoPrefixMatches(values: string[], prefix: string) { expect(values.filter((value) => value.startsWith(prefix))).toEqual([]); @@ -201,6 +205,143 @@ describe("bundled plugin build entries", () => { } }); + it("builds explicitly selected external plugins only for Docker", () => { + const baselineEnv = { ...process.env }; + delete baselineEnv[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]; + const dockerEnv = { + ...baselineEnv, + [DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]: "slack clickclack,slack,msteams", + }; + const entries = listBundledPluginBuildEntries({ env: dockerEnv }); + const baselineArtifacts = listBundledPluginPackArtifacts({ env: baselineEnv }); + const artifacts = listBundledPluginPackArtifacts({ env: dockerEnv }); + const reorderedEntries = listBundledPluginBuildEntries({ + env: { + ...baselineEnv, + [DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]: "msteams,clickclack slack", + }, + }); + const entryKeys = Object.keys(entries); + + expect(entries["extensions/clickclack/index"]).toBe("extensions/clickclack/index.ts"); + expect(entries["extensions/slack/index"]).toBe("extensions/slack/index.ts"); + expect(entries["extensions/slack/setup-entry"]).toBe("extensions/slack/setup-entry.ts"); + expect(entries["extensions/msteams/index"]).toBe("extensions/msteams/index.ts"); + expect(entries["extensions/clawrouter/index"]).toBe("extensions/clawrouter/index.ts"); + expect(entryKeys.findIndex((entry) => entry.startsWith("extensions/clickclack/"))).toBeLessThan( + entryKeys.findIndex((entry) => entry.startsWith("extensions/slack/")), + ); + expect(Object.keys(reorderedEntries)).toEqual(entryKeys); + expect(artifacts).toEqual(baselineArtifacts); + expectNoPrefixMatches(artifacts, "dist/extensions/clickclack/"); + expectNoPrefixMatches(artifacts, "dist/extensions/msteams/"); + expectNoPrefixMatches(artifacts, "dist/extensions/slack/"); + }); + + it("sorts Docker-selected build entries without git metadata", () => { + const repoDir = tempDirs.make("openclaw-docker-build-entries-"); + const extensionsDir = path.join(repoDir, "extensions"); + + for (const pluginId of ["clickclack", "msteams", "slack"]) { + const pluginDir = path.join(extensionsDir, pluginId); + fs.mkdirSync(pluginDir, { recursive: true }); + fs.writeFileSync(path.join(pluginDir, "index.ts"), "export default {};\n"); + fs.writeFileSync( + path.join(pluginDir, "openclaw.plugin.json"), + `${JSON.stringify({ id: pluginId })}\n`, + ); + fs.writeFileSync( + path.join(pluginDir, "package.json"), + `${JSON.stringify({ + name: `@openclaw/${pluginId}`, + openclaw: { + extensions: ["./index.ts"], + build: { bundledDist: false }, + }, + })}\n`, + ); + } + + const unsortedDirents = fs.readdirSync(extensionsDir, { withFileTypes: true }).toReversed(); + const readdirSpy = vi + .spyOn(fs, "readdirSync") + .mockImplementationOnce(() => unsortedDirents as never); + try { + expect( + Object.keys( + listBundledPluginBuildEntries({ + cwd: repoDir, + env: { + ...process.env, + [DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]: "slack,msteams,clickclack", + }, + }), + ), + ).toEqual([ + "extensions/clickclack/index", + "extensions/msteams/index", + "extensions/slack/index", + ]); + } finally { + readdirSpy.mockRestore(); + } + }); + + it("preserves known dependency-only Docker plugin selections", () => { + const baselineEnv = { ...process.env }; + delete baselineEnv[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]; + const baselineEntries = listBundledPluginBuildEntries({ env: baselineEnv }); + const selectedEntries = listBundledPluginBuildEntries({ + env: { + ...baselineEnv, + [DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]: "whatsapp,qqbot", + }, + }); + + expect(selectedEntries).toEqual(baselineEntries); + expectNoPrefixMatches(Object.keys(selectedEntries), "extensions/qqbot/"); + expectNoPrefixMatches(Object.keys(selectedEntries), "extensions/whatsapp/"); + }); + + it("preserves known package-less bundled Docker plugin selections", () => { + const baselineEnv = { ...process.env }; + delete baselineEnv[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]; + const baselineEntries = listBundledPluginBuildEntries({ env: baselineEnv }); + const selectedEntries = listBundledPluginBuildEntries({ + env: { + ...baselineEnv, + [DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]: "active-memory", + }, + }); + + expect(selectedEntries).toEqual(baselineEntries); + expect(selectedEntries["extensions/active-memory/index"]).toBe( + "extensions/active-memory/index.ts", + ); + }); + + it("rejects unknown and invalid Docker plugin selections", () => { + for (const [selection, message] of [ + [ + "missing-plugin", + `${DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV} references unknown plugin id(s): missing-plugin`, + ], + [ + "../clickclack", + `${DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV} contains invalid plugin id(s): ../clickclack`, + ], + ] as const) { + expect(() => + listBundledPluginBuildEntries({ + env: { + ...process.env, + [DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV]: selection, + }, + }), + ).toThrow(message); + } + }); + it("keeps Cohere bundled through the externalization transition", () => { const artifacts = listBundledPluginPackArtifacts(); diff --git a/test/scripts/docker-plugin-selection.test.ts b/test/scripts/docker-plugin-selection.test.ts new file mode 100644 index 000000000000..4ccbddb786fe --- /dev/null +++ b/test/scripts/docker-plugin-selection.test.ts @@ -0,0 +1,62 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const repoRoot = path.resolve(fileURLToPath(new URL("../..", import.meta.url))); +const selectorScript = path.join(repoRoot, "scripts/lib/docker-plugin-selection.mjs"); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +function writePlugin(extensionsRoot: string, dirName: string, manifestId?: string) { + const pluginDir = path.join(extensionsRoot, dirName); + fs.mkdirSync(pluginDir, { recursive: true }); + fs.writeFileSync(path.join(pluginDir, "package.json"), `${JSON.stringify({ name: dirName })}\n`); + if (manifestId) { + fs.writeFileSync( + path.join(pluginDir, "openclaw.plugin.json"), + `${JSON.stringify({ id: manifestId })}\n`, + ); + } +} + +function runSelector(extensionsRoot: string, selection: string) { + return spawnSync(process.execPath, [selectorScript, extensionsRoot, selection], { + encoding: "utf8", + }); +} + +describe("Docker plugin selection", () => { + it("resolves manifest ids and source directory names deterministically", () => { + const extensionsRoot = tempDirs.make("openclaw-docker-plugin-selection-"); + writePlugin(extensionsRoot, "source-only"); + writePlugin(extensionsRoot, "provider-source", "provider-id"); + + const result = runSelector( + extensionsRoot, + "source-only,provider-id provider-source,provider-id", + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toBe("provider-source\nsource-only\n"); + }); + + it("fails closed for unknown, invalid, and ambiguous ids", () => { + const extensionsRoot = tempDirs.make("openclaw-docker-plugin-selection-errors-"); + writePlugin(extensionsRoot, "shared"); + writePlugin(extensionsRoot, "other-source", "shared"); + + for (const [selection, message] of [ + ["missing-plugin", "unknown OPENCLAW_EXTENSIONS plugin id: missing-plugin"], + ["../invalid", "invalid OPENCLAW_EXTENSIONS plugin id: ../invalid"], + ["shared", "ambiguous OPENCLAW_EXTENSIONS plugin id: shared"], + ] as const) { + const result = runSelector(extensionsRoot, selection); + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain(message); + } + }); +});