build: exclude private QA from npm package

This commit is contained in:
Peter Steinberger
2026-04-15 09:38:45 -07:00
parent 78ac118427
commit 229eb72cf6
30 changed files with 539 additions and 86 deletions

View File

@@ -1,6 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { NON_PACKAGED_BUNDLED_PLUGIN_DIRS } from "./lib/bundled-plugin-build-entries.mjs";
import { shouldBuildBundledCluster } from "./lib/optional-bundled-clusters.mjs";
import {
removeFileIfExists,
@@ -12,6 +13,13 @@ const GENERATED_BUNDLED_SKILLS_DIR = "bundled-skills";
const TRANSIENT_COPY_ERROR_CODES = new Set(["EEXIST", "ENOENT", "ENOTEMPTY", "EBUSY"]);
const COPY_RETRY_DELAYS_MS = [10, 25, 50];
function shouldCopyBundledPluginMetadata(id, env) {
if (!NON_PACKAGED_BUNDLED_PLUGIN_DIRS.has(id)) {
return true;
}
return env.OPENCLAW_BUILD_PRIVATE_QA === "1";
}
export function rewritePackageExtensions(entries) {
if (!Array.isArray(entries)) {
return undefined;
@@ -234,6 +242,10 @@ export function copyBundledPluginMetadata(params = {}) {
? JSON.parse(fs.readFileSync(packageJsonPath, "utf8"))
: undefined;
const topLevelPublicSurfaceEntries = collectTopLevelPublicSurfaceEntries(pluginDir);
if (!shouldCopyBundledPluginMetadata(dirent.name, env)) {
removePathIfExists(distPluginDir);
continue;
}
if (!shouldBuildBundledCluster(dirent.name, env, { packageJson })) {
removePathIfExists(distPluginDir);
continue;

View File

@@ -10,6 +10,7 @@ export type BundledPluginBuildEntryParams = {
env?: NodeJS.ProcessEnv;
};
export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS: Set<string>;
export function collectBundledPluginBuildEntries(
params?: BundledPluginBuildEntryParams,
): BundledPluginBuildEntry[];

View File

@@ -10,6 +10,7 @@ export type BundledPluginBuildEntryParams = {
env?: NodeJS.ProcessEnv;
};
export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS: Set<string>;
export function collectBundledPluginBuildEntries(
params?: BundledPluginBuildEntryParams,
): BundledPluginBuildEntry[];

View File

@@ -8,7 +8,7 @@ import {
import { shouldBuildBundledCluster } from "./optional-bundled-clusters.mjs";
const TOP_LEVEL_PUBLIC_SURFACE_EXTENSIONS = new Set([".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"]);
const NON_PACKAGED_BUNDLED_PLUGIN_DIRS = new Set(["qa-channel", "qa-lab", "qa-matrix"]);
export const NON_PACKAGED_BUNDLED_PLUGIN_DIRS = new Set(["qa-channel", "qa-lab", "qa-matrix"]);
const toPosixPath = (value) => value.replaceAll("\\", "/");
function readBundledPluginPackageJson(packageJsonPath) {

View File

@@ -179,7 +179,6 @@
"matrix-runtime-surface",
"matrix-surface",
"matrix-thread-bindings",
"qa-runtime",
"qa-runner-runtime",
"mattermost",
"mattermost-policy",

View File

@@ -46,7 +46,6 @@ const LEGACY_CONTEXT_ENGINE_UNRESOLVED_RUNTIME_MARKER =
"Failed to load legacy context engine runtime.";
const LEGACY_UPDATE_COMPAT_RUNTIME_SIDECAR_PATHS = [
"dist/extensions/qa-channel/runtime-api.js",
"dist/extensions/qa-lab/runtime-api.js",
] as const;
const PUBLISHED_BUNDLED_RUNTIME_SIDECAR_PATHS = [
...BUNDLED_RUNTIME_SIDECAR_PATHS.filter((relativePath) =>

View File

@@ -2,8 +2,12 @@
import { execFileSync } from "node:child_process";
import { readFileSync } from "node:fs";
import { basename } from "node:path";
import { basename, join } from "node:path";
import { pathToFileURL } from "node:url";
import {
PACKAGE_DIST_INVENTORY_RELATIVE_PATH,
writePackageDistInventory,
} from "../src/infra/package-dist-inventory.ts";
import {
compareReleaseVersions as compareReleaseVersionsBase,
resolveNpmDistTagMirrorAuth as resolveNpmDistTagMirrorAuthBase,
@@ -55,11 +59,9 @@ export type NpmDistTagMirrorAuth = {
};
const EXPECTED_REPOSITORY_URL = "https://github.com/openclaw/openclaw";
const MAX_CALVER_DISTANCE_DAYS = 2;
const LEGACY_UPDATE_COMPAT_PACKED_PATHS = [
"dist/extensions/qa-channel/runtime-api.js",
"dist/extensions/qa-lab/runtime-api.js",
] as const;
const LEGACY_UPDATE_COMPAT_PACKED_PATHS = ["dist/extensions/qa-channel/runtime-api.js"] as const;
const REQUIRED_PACKED_PATHS = [
PACKAGE_DIST_INVENTORY_RELATIVE_PATH,
"dist/control-ui/index.html",
...LEGACY_UPDATE_COMPAT_PACKED_PATHS,
...WORKSPACE_TEMPLATE_PACK_PATHS,
@@ -81,7 +83,28 @@ const FORBIDDEN_PACKED_PATH_RULES = [
describe: (packedPath: string) =>
`npm package must not include private QA lab artifact "${packedPath}".`,
},
{
prefix: "dist/plugin-sdk/extensions/qa-lab/",
describe: (packedPath: string) =>
`npm package must not include private QA lab type artifact "${packedPath}".`,
},
{
prefix: "dist/qa-runtime-",
describe: (packedPath: string) =>
`npm package must not include private QA runtime chunk "${packedPath}".`,
},
{
prefix: "qa/",
describe: (packedPath: string) =>
`npm package must not include private QA suite artifact "${packedPath}".`,
},
] as const;
const FORBIDDEN_PRIVATE_QA_CONTENT_MARKERS = [
"//#region extensions/qa-lab/",
"qa-lab/cli.js",
"qa-lab/runtime-api.js",
] as const;
const FORBIDDEN_PRIVATE_QA_CONTENT_SCAN_PREFIXES = ["dist/"] as const;
const NPM_PACK_MAX_BUFFER_BYTES = 64 * 1024 * 1024;
const skipPackValidationEnv = "OPENCLAW_NPM_RELEASE_SKIP_PACK_CHECK";
@@ -466,6 +489,7 @@ function collectPackedTarballErrors(): string[] {
return [
...collectControlUiPackErrors(packedPaths),
...collectForbiddenPackedPathErrors(packedPaths),
...collectForbiddenPackedContentErrors(packedPaths),
];
}
@@ -486,7 +510,41 @@ export function collectForbiddenPackedPathErrors(paths: Iterable<string>): strin
return errors.toSorted((left, right) => left.localeCompare(right));
}
function main(): number {
export function collectForbiddenPackedContentErrors(
paths: Iterable<string>,
rootDir = process.cwd(),
): string[] {
const textPathPattern = /\.(?:[cm]?js|d\.ts|json|md|mjs|cjs)$/u;
const errors: string[] = [];
for (const packedPath of paths) {
if (
!FORBIDDEN_PRIVATE_QA_CONTENT_SCAN_PREFIXES.some((prefix) => packedPath.startsWith(prefix))
) {
continue;
}
if (!textPathPattern.test(packedPath)) {
continue;
}
let content: string;
try {
content = readFileSync(pathToFileURL(join(rootDir, packedPath)), "utf8");
} catch {
continue;
}
const matchedMarker = FORBIDDEN_PRIVATE_QA_CONTENT_MARKERS.find((marker) =>
content.includes(marker),
);
if (!matchedMarker) {
continue;
}
errors.push(
`npm package must not include private QA lab marker "${matchedMarker}" in "${packedPath}".`,
);
}
return errors.toSorted((left, right) => left.localeCompare(right));
}
async function main(): Promise<number> {
const pkg = loadPackageJson();
const now = new Date();
const skipPackValidation = shouldSkipPackedTarballValidation();
@@ -498,6 +556,9 @@ function main(): number {
releaseMainRef: process.env.RELEASE_MAIN_REF,
now,
});
if (!skipPackValidation) {
await writePackageDistInventory(process.cwd());
}
const tarballErrors = skipPackValidation ? [] : collectPackedTarballErrors();
const errors = [...metadataErrors, ...tagErrors, ...tarballErrors];
@@ -519,5 +580,5 @@ function main(): number {
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
process.exit(main());
process.exit(await main());
}

View File

@@ -12,6 +12,7 @@ import {
closeSync,
existsSync,
lstatSync,
mkdirSync,
openSync,
readdirSync,
readFileSync,
@@ -33,6 +34,14 @@ const DEFAULT_EXTENSIONS_DIR = join(__dirname, "..", "dist", "extensions");
const DEFAULT_PACKAGE_ROOT = join(__dirname, "..");
const DISABLE_POSTINSTALL_ENV = "OPENCLAW_DISABLE_BUNDLED_PLUGIN_POSTINSTALL";
const DIST_INVENTORY_PATH = "dist/postinstall-inventory.json";
const LEGACY_UPDATE_COMPAT_SIDECARS = [
{
path: "dist/extensions/qa-lab/runtime-api.js",
removedPrefix: "dist/extensions/qa-lab/",
content:
"// Compatibility stub for older OpenClaw updaters. QA Lab is not packaged.\nexport {};\n",
},
];
const BAILEYS_MEDIA_FILE = join(
"node_modules",
"@whiskeysockets",
@@ -296,6 +305,30 @@ export function pruneInstalledPackageDist(params = {}) {
return removed;
}
export function restoreLegacyUpdaterCompatSidecars(params = {}) {
const packageRoot = params.packageRoot ?? DEFAULT_PACKAGE_ROOT;
const writeFile = params.writeFileSync ?? writeFileSync;
const makeDirectory = params.mkdirSync ?? mkdirSync;
const log = params.log ?? console;
const restored = [];
for (const sidecar of LEGACY_UPDATE_COMPAT_SIDECARS) {
// Older npm updater builds verify this exact sidecar after npm has already
// replaced the package. npm may remove stale QA Lab files before this
// postinstall hook runs, so this must be generated independently of prune
// results. The tarball and dist inventory still omit QA Lab.
const sidecarPath = join(packageRoot, sidecar.path);
makeDirectory(dirname(sidecarPath), { recursive: true });
writeFile(sidecarPath, sidecar.content, "utf8");
restored.push(sidecar.path);
}
if (restored.length > 0) {
log.log(`[postinstall] restored legacy updater compat sidecars: ${restored.join(", ")}`);
}
return restored;
}
function dependencySentinelPath(depName) {
return join("node_modules", ...depName.split("/"), "package.json");
}
@@ -650,7 +683,7 @@ export function runBundledPluginPostinstall(params = {}) {
});
return;
}
pruneInstalledPackageDist({
const prunedDistFiles = pruneInstalledPackageDist({
packageRoot,
existsSync: pathExists,
readFileSync: params.readFileSync,
@@ -658,6 +691,13 @@ export function runBundledPluginPostinstall(params = {}) {
rmSync: params.rmSync,
log,
});
restoreLegacyUpdaterCompatSidecars({
packageRoot,
removedFiles: prunedDistFiles,
mkdirSync: params.mkdirSync,
writeFileSync: params.writeFileSync,
log,
});
if (
!shouldRunBundledPluginPostinstall({
env,

View File

@@ -5,6 +5,10 @@ import { mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import {
PACKAGE_DIST_INVENTORY_RELATIVE_PATH,
writePackageDistInventory,
} from "../src/infra/package-dist-inventory.ts";
import {
collectBundledExtensionManifestErrors,
type BundledExtension,
@@ -38,13 +42,13 @@ type PackFile = { path: string };
type PackResult = { files?: PackFile[]; filename?: string; unpackedSize?: number };
const requiredPathGroups = [
PACKAGE_DIST_INVENTORY_RELATIVE_PATH,
["dist/index.js", "dist/index.mjs"],
["dist/entry.js", "dist/entry.mjs"],
...listPluginSdkDistArtifacts(),
...listBundledPluginPackArtifacts(),
...listStaticExtensionAssetOutputs(),
...WORKSPACE_TEMPLATE_PACK_PATHS,
...listRequiredQaScenarioPackPaths(),
"scripts/npm-runner.mjs",
"scripts/preinstall-package-manager-warning.mjs",
"scripts/postinstall-bundled-plugins.mjs",
@@ -57,21 +61,27 @@ const requiredPathGroups = [
const forbiddenPrefixes = [
"dist-runtime/",
"dist/OpenClaw.app/",
"dist/extensions/qa-lab/",
"dist/plugin-sdk/extensions/qa-lab/",
"dist/plugin-sdk/qa-lab.",
"dist/plugin-sdk/qa-runtime.",
"dist/plugin-sdk/src/plugin-sdk/qa-lab.d.ts",
"dist/plugin-sdk/src/plugin-sdk/qa-runtime.d.ts",
"dist/qa-runtime-",
"dist/plugin-sdk/.tsbuildinfo",
"docs/.generated/",
"qa/",
];
const forbiddenPrivateQaContentMarkers = [
"//#region extensions/qa-lab/",
"qa-lab/cli.js",
"qa-lab/runtime-api.js",
] as const;
const forbiddenPrivateQaContentScanPrefixes = ["dist/"] as const;
const appcastPath = resolve("appcast.xml");
const laneBuildMin = 1_000_000_000;
const laneFloorAdoptionDateKey = 20260227;
export function listRequiredQaScenarioPackPaths(): string[] {
const scenariosDir = resolve("qa/scenarios");
return readdirSync(scenariosDir, { withFileTypes: true })
.filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
.map((entry) => `qa/scenarios/${entry.name}`)
.toSorted((left, right) => left.localeCompare(right));
}
function collectBundledExtensions(): BundledExtension[] {
const extensionsDir = resolve("extensions");
const entries = readdirSync(extensionsDir, { withFileTypes: true }).filter((entry) =>
@@ -269,6 +279,30 @@ export function collectForbiddenPackPaths(paths: Iterable<string>): string[] {
.toSorted((left, right) => left.localeCompare(right));
}
export function collectForbiddenPackContentPaths(
paths: Iterable<string>,
rootDir = process.cwd(),
): string[] {
const textPathPattern = /\.(?:[cm]?js|d\.ts|json|md|mjs|cjs)$/u;
return [...paths]
.filter((packedPath) => {
if (!forbiddenPrivateQaContentScanPrefixes.some((prefix) => packedPath.startsWith(prefix))) {
return false;
}
if (!textPathPattern.test(packedPath)) {
return false;
}
let content: string;
try {
content = readFileSync(resolve(rootDir, packedPath), "utf8");
} catch {
return false;
}
return forbiddenPrivateQaContentMarkers.some((marker) => content.includes(marker));
})
.toSorted((left, right) => left.localeCompare(right));
}
export { collectPackUnpackedSizeErrors } from "./lib/npm-pack-budget.mjs";
function extractTag(item: string, tag: string): string | null {
@@ -430,6 +464,7 @@ async function main() {
checkAppcastSparkleVersions();
await checkPluginSdkExports();
checkBundledExtensionMetadata();
await writePackageDistInventory(process.cwd());
const results = runPackDry();
const files = results.flatMap((entry) => entry.files ?? []);
@@ -444,9 +479,15 @@ async function main() {
})
.toSorted((left, right) => left.localeCompare(right));
const forbidden = collectForbiddenPackPaths(paths);
const forbiddenContent = collectForbiddenPackContentPaths(paths);
const sizeErrors = collectNpmPackUnpackedSizeErrors(results);
if (missing.length > 0 || forbidden.length > 0 || sizeErrors.length > 0) {
if (
missing.length > 0 ||
forbidden.length > 0 ||
forbiddenContent.length > 0 ||
sizeErrors.length > 0
) {
if (missing.length > 0) {
console.error("release-check: missing files in npm pack:");
for (const path of missing) {
@@ -471,6 +512,12 @@ async function main() {
console.error(` - ${path}`);
}
}
if (forbiddenContent.length > 0) {
console.error("release-check: forbidden private QA markers in npm pack:");
for (const path of forbiddenContent) {
console.error(` - ${path}`);
}
}
if (sizeErrors.length > 0) {
console.error("release-check: npm pack unpacked size budget exceeded:");
for (const error of sizeErrors) {

View File

@@ -208,6 +208,13 @@ export const resolveBuildRequirement = (deps) => {
if (deps.env.OPENCLAW_FORCE_BUILD === "1") {
return { shouldBuild: true, reason: "force_build" };
}
if (
deps.env.OPENCLAW_BUILD_PRIVATE_QA === "1" &&
deps.privateQaDistEntry &&
statMtime(deps.privateQaDistEntry, deps.fs) == null
) {
return { shouldBuild: true, reason: "missing_private_qa_dist" };
}
const stamp = readBuildStamp(deps);
if (stamp.mtime == null) {
return { shouldBuild: true, reason: "missing_build_stamp" };
@@ -255,6 +262,7 @@ const BUILD_REASON_LABELS = {
git_head_changed: "git head changed",
dirty_watched_tree: "dirty watched source tree",
source_mtime_newer: "source mtime newer than build stamp",
missing_private_qa_dist: "private QA dist entry missing",
clean: "clean",
};
@@ -389,6 +397,11 @@ export async function runNodeMain(params = {}) {
path: path.join(deps.cwd, sourceRoot),
}));
deps.configFiles = runNodeConfigFiles.map((filePath) => path.join(deps.cwd, filePath));
deps.privateQaDistEntry = path.join(deps.distRoot, "extensions", "qa-lab", "cli.js");
if (deps.args[0] === "qa") {
deps.env.OPENCLAW_BUILD_PRIVATE_QA = "1";
deps.env.OPENCLAW_ENABLE_PRIVATE_QA_CLI = "1";
}
const buildRequirement = resolveBuildRequirement(deps);
if (!buildRequirement.shouldBuild) {

View File

@@ -228,6 +228,7 @@ prepare_update_tarball() {
UPDATE_EXPECT_VERSION="$(
node -p 'JSON.parse(require("node:fs").readFileSync("package.json", "utf8")).version'
)"
node --import tsx scripts/write-package-dist-inventory.ts
quiet_npm pack --ignore-scripts --json --pack-destination "$UPDATE_DIR" >"$pack_json_file"
fi
UPDATE_TGZ_FILE="$(

View File

@@ -0,0 +1,12 @@
#!/usr/bin/env -S node --import tsx
import { pathToFileURL } from "node:url";
import { writePackageDistInventory } from "../src/infra/package-dist-inventory.ts";
export async function writeCurrentPackageDistInventory(): Promise<void> {
await writePackageDistInventory(process.cwd());
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
await writeCurrentPackageDistInventory();
}