fix: serialize bundled runtime dependency repair

This commit is contained in:
Peter Steinberger
2026-04-24 20:29:11 +01:00
parent def392ad7d
commit 56e299cbca
6 changed files with 306 additions and 66 deletions

View File

@@ -47,6 +47,11 @@ Benefits:
- Prefer a first-party OpenClaw surface when the task is fundamentally "run inference."
- Use the normal local path without requiring the gateway for most infer commands.
For end-to-end provider checks, prefer `openclaw infer ...` once lower-level
provider tests are green. It exercises the shipped CLI, config loading,
default-agent resolution, bundled plugin activation, runtime-dependency repair,
and the shared capability runtime before the provider request is made.
## Command tree
```text
@@ -157,6 +162,25 @@ openclaw infer image describe --file ./photo.jpg --model ollama/qwen2.5vl:7b --j
Notes:
- Use `image edit` when starting from existing input files.
- Use `image providers --json` to verify which bundled image providers are
discoverable, configured, selected, and which generation/edit capabilities
each provider exposes.
- Use `image generate --model <provider/model> --json` as the narrowest live
CLI smoke for image generation changes. Example:
```bash
openclaw infer image providers --json
openclaw infer image generate \
--model google/gemini-3.1-flash-image-preview \
--prompt "Minimal flat test image: one blue square on a white background, no text." \
--output ./openclaw-infer-image-smoke.png \
--json
```
The JSON response reports `ok`, `provider`, `model`, `attempts`, and written
output paths. When `--output` is set, the final extension may follow the
provider's returned MIME type.
- For `image describe`, `--model` must be an image-capable `<provider/model>`.
- For local Ollama vision models, pull the model first and set `OLLAMA_API_KEY` to any placeholder value, for example `ollama-local`. See [Ollama](/providers/ollama#vision-and-image-description).
@@ -258,6 +282,10 @@ Top-level fields are stable:
- `outputs`
- `error`
For generated media commands, `outputs` contains files written by OpenClaw. Use
the `path`, `mimeType`, `size`, and any media-specific dimensions in that array
for automation instead of parsing human-readable stdout.
## Common pitfalls
```bash

View File

@@ -387,6 +387,12 @@ are missing, doctor reports the packages and installs them in
use `openclaw plugins install` / `openclaw plugins update`; doctor does not
install dependencies for arbitrary plugin paths.
The Gateway and local CLI can also repair active bundled plugin runtime
dependencies on demand before importing a bundled plugin. These installs are
scoped to the plugin runtime install root, run with scripts disabled, do not
write a package lock, and are guarded by an install-root lock so concurrent CLI
or Gateway starts do not mutate the same `node_modules` tree at the same time.
### 8) Gateway service migrations and cleanup hints
Doctor detects legacy gateway services (launchd/systemd/schtasks) and

View File

@@ -402,11 +402,9 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local
- Loads missing provider env vars from your login shell (`~/.profile`) before probing
- Uses live/env API keys ahead of stored auth profiles by default, so stale test keys in `auth-profiles.json` do not mask real shell credentials
- Skips providers with no usable auth/profile/model
- Runs the stock image-generation variants through the shared runtime capability:
- `google:flash-generate`
- `google:pro-generate`
- `google:pro-edit`
- `openai:default-generate`
- Runs each configured provider through the shared image-generation runtime:
- `<provider>:generate`
- `<provider>:edit` when the provider declares edit support
- Current bundled providers covered:
- `fal`
- `google`
@@ -422,6 +420,23 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local
- Optional auth behavior:
- `OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS=1` to force profile-store auth and ignore env-only overrides
For the shipped CLI path, add an `infer` smoke after the provider/runtime live
test passes:
```bash
OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_INFER_CLI_TEST=1 pnpm test:live -- test/image-generation.infer-cli.live.test.ts
openclaw infer image providers --json
openclaw infer image generate \
--model google/gemini-3.1-flash-image-preview \
--prompt "Minimal flat test image: one blue square on a white background, no text." \
--output ./openclaw-infer-image-smoke.png \
--json
```
This covers CLI argument parsing, config/default-agent resolution, bundled
plugin activation, on-demand bundled runtime-dependency repair, the shared
image-generation runtime, and the live provider request.
## Music generation live
- Test: `extensions/music-generation-providers.live.test.ts`

View File

@@ -656,6 +656,50 @@ describe("ensureBundledPluginRuntimeDeps", () => {
]);
});
it("removes stale runtime-deps install locks before repairing deps", () => {
const packageRoot = makeTempDir();
const pluginRoot = path.join(packageRoot, "dist", "extensions", "openai");
fs.mkdirSync(pluginRoot, { recursive: true });
fs.writeFileSync(
path.join(pluginRoot, "package.json"),
JSON.stringify({
dependencies: {
"@mariozechner/pi-ai": "0.70.2",
},
}),
);
const lockDir = path.join(pluginRoot, ".openclaw-runtime-deps.lock");
fs.mkdirSync(lockDir, { recursive: true });
fs.writeFileSync(
path.join(lockDir, "owner.json"),
JSON.stringify({ pid: process.pid, createdAtMs: 0 }),
);
const calls: BundledRuntimeDepsInstallParams[] = [];
const result = ensureBundledPluginRuntimeDeps({
env: {},
installDeps: (params) => {
calls.push(params);
fs.mkdirSync(path.join(params.installRoot, "node_modules", "@mariozechner", "pi-ai"), {
recursive: true,
});
fs.writeFileSync(
path.join(params.installRoot, "node_modules", "@mariozechner", "pi-ai", "package.json"),
JSON.stringify({ name: "@mariozechner/pi-ai", version: "0.70.2" }),
);
},
pluginId: "openai",
pluginRoot,
});
expect(result).toEqual({
installedSpecs: ["@mariozechner/pi-ai@0.70.2"],
retainSpecs: ["@mariozechner/pi-ai@0.70.2"],
});
expect(calls).toHaveLength(1);
expect(fs.existsSync(lockDir)).toBe(false);
});
it("does not install when runtime deps are only workspace links", () => {
const packageRoot = makeTempDir();
const extensionsRoot = path.join(packageRoot, "dist", "extensions");

View File

@@ -49,6 +49,11 @@ const RETAINED_RUNTIME_DEPS_MANIFEST = ".openclaw-runtime-deps.json";
// to the plugin root. Source-checkout installs already have their own cache
// path and keep using it.
const PLUGIN_ROOT_INSTALL_STAGE_DIR = ".openclaw-install-stage";
const BUNDLED_RUNTIME_DEPS_LOCK_DIR = ".openclaw-runtime-deps.lock";
const BUNDLED_RUNTIME_DEPS_LOCK_OWNER_FILE = "owner.json";
const BUNDLED_RUNTIME_DEPS_LOCK_WAIT_MS = 100;
const BUNDLED_RUNTIME_DEPS_LOCK_TIMEOUT_MS = 5 * 60_000;
const BUNDLED_RUNTIME_DEPS_LOCK_STALE_MS = 10 * 60_000;
export type BundledRuntimeDepsNpmRunner = {
command: string;
@@ -170,6 +175,82 @@ function readJsonObject(filePath: string): JsonObject | null {
}
}
function sleepSync(ms: number): void {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
}
function isProcessAlive(pid: number): boolean {
if (!Number.isInteger(pid) || pid <= 0) {
return false;
}
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function readRuntimeDepsLockOwner(lockDir: string): { pid?: number; createdAtMs?: number } {
const owner = readJsonObject(path.join(lockDir, BUNDLED_RUNTIME_DEPS_LOCK_OWNER_FILE));
return {
pid: typeof owner?.pid === "number" ? owner.pid : undefined,
createdAtMs: typeof owner?.createdAtMs === "number" ? owner.createdAtMs : undefined,
};
}
function removeRuntimeDepsLockIfStale(lockDir: string, nowMs: number): boolean {
const owner = readRuntimeDepsLockOwner(lockDir);
const createdAtMs = owner.createdAtMs;
const staleByTime =
typeof createdAtMs === "number" && nowMs - createdAtMs > BUNDLED_RUNTIME_DEPS_LOCK_STALE_MS;
const staleByPid = typeof owner.pid === "number" && !isProcessAlive(owner.pid);
if (!staleByTime && !staleByPid) {
return false;
}
try {
fs.rmSync(lockDir, { recursive: true, force: true });
return true;
} catch {
return false;
}
}
function withBundledRuntimeDepsInstallRootLock<T>(installRoot: string, run: () => T): T {
fs.mkdirSync(installRoot, { recursive: true });
const lockDir = path.join(installRoot, BUNDLED_RUNTIME_DEPS_LOCK_DIR);
const startedAt = Date.now();
let locked = false;
while (!locked) {
try {
fs.mkdirSync(lockDir);
fs.writeFileSync(
path.join(lockDir, BUNDLED_RUNTIME_DEPS_LOCK_OWNER_FILE),
`${JSON.stringify({ pid: process.pid, createdAtMs: Date.now() }, null, 2)}\n`,
"utf8",
);
locked = true;
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code !== "EEXIST") {
throw error;
}
removeRuntimeDepsLockIfStale(lockDir, Date.now());
if (Date.now() - startedAt > BUNDLED_RUNTIME_DEPS_LOCK_TIMEOUT_MS) {
throw new Error(`Timed out waiting for bundled runtime deps lock at ${lockDir}`, {
cause: error,
});
}
sleepSync(BUNDLED_RUNTIME_DEPS_LOCK_WAIT_MS);
}
}
try {
return run();
} finally {
fs.rmSync(lockDir, { recursive: true, force: true });
}
}
function collectRuntimeDeps(packageJson: JsonObject): Record<string, unknown> {
return {
...(packageJson.dependencies as Record<string, unknown> | undefined),
@@ -935,67 +1016,69 @@ export function ensureBundledPluginRuntimeDeps(params: {
const installRoot = resolveBundledRuntimeDependencyInstallRoot(params.pluginRoot, {
env: params.env,
});
const persistRetainedManifest = shouldPersistRetainedRuntimeDepsManifest({
pluginRoot: params.pluginRoot,
installRoot,
});
if (!persistRetainedManifest) {
removeRetainedRuntimeDepsManifest(installRoot);
}
const dependencySpecs = deps
.map((dep) => `${dep.name}@${dep.version}`)
.toSorted((left, right) => left.localeCompare(right));
const missingSpecs = deps
.filter((dep) => !hasDependencySentinel([installRoot], dep))
.map((dep) => `${dep.name}@${dep.version}`)
.toSorted((left, right) => left.localeCompare(right));
if (missingSpecs.length === 0) {
return { installedSpecs: [], retainSpecs: [] };
}
const retainedManifestSpecs = persistRetainedManifest
? readRetainedRuntimeDepsManifest(installRoot)
: [];
const installSpecs = [
...new Set([...(params.retainSpecs ?? []), ...retainedManifestSpecs, ...dependencySpecs]),
].toSorted((left, right) => left.localeCompare(right));
const cacheDir = resolveSourceCheckoutRuntimeDepsCacheDir({
pluginId: params.pluginId,
pluginRoot: params.pluginRoot,
installSpecs,
});
const isPluginRootInstall = path.resolve(installRoot) === path.resolve(params.pluginRoot);
const sourceCheckoutCacheStage =
cacheDir &&
isPluginRootInstall &&
resolveSourceCheckoutBundledPluginPackageRoot(params.pluginRoot)
? cacheDir
: undefined;
const installExecutionRoot =
sourceCheckoutCacheStage ??
(isPluginRootInstall ? path.join(installRoot, PLUGIN_ROOT_INSTALL_STAGE_DIR) : undefined);
if (
restoreSourceCheckoutRuntimeDepsFromCache({
cacheDir,
deps,
return withBundledRuntimeDepsInstallRootLock(installRoot, () => {
const persistRetainedManifest = shouldPersistRetainedRuntimeDepsManifest({
pluginRoot: params.pluginRoot,
installRoot,
})
) {
return { installedSpecs: [], retainSpecs: [] };
}
});
if (!persistRetainedManifest) {
removeRetainedRuntimeDepsManifest(installRoot);
}
const dependencySpecs = deps
.map((dep) => `${dep.name}@${dep.version}`)
.toSorted((left, right) => left.localeCompare(right));
const missingSpecs = deps
.filter((dep) => !hasDependencySentinel([installRoot], dep))
.map((dep) => `${dep.name}@${dep.version}`)
.toSorted((left, right) => left.localeCompare(right));
if (missingSpecs.length === 0) {
return { installedSpecs: [], retainSpecs: [] };
}
const retainedManifestSpecs = persistRetainedManifest
? readRetainedRuntimeDepsManifest(installRoot)
: [];
const installSpecs = [
...new Set([...(params.retainSpecs ?? []), ...retainedManifestSpecs, ...dependencySpecs]),
].toSorted((left, right) => left.localeCompare(right));
const cacheDir = resolveSourceCheckoutRuntimeDepsCacheDir({
pluginId: params.pluginId,
pluginRoot: params.pluginRoot,
installSpecs,
});
const isPluginRootInstall = path.resolve(installRoot) === path.resolve(params.pluginRoot);
const sourceCheckoutCacheStage =
cacheDir &&
isPluginRootInstall &&
resolveSourceCheckoutBundledPluginPackageRoot(params.pluginRoot)
? cacheDir
: undefined;
const installExecutionRoot =
sourceCheckoutCacheStage ??
(isPluginRootInstall ? path.join(installRoot, PLUGIN_ROOT_INSTALL_STAGE_DIR) : undefined);
if (
restoreSourceCheckoutRuntimeDepsFromCache({
cacheDir,
deps,
installRoot,
})
) {
return { installedSpecs: [], retainSpecs: [] };
}
const install =
params.installDeps ??
((installParams) =>
installBundledRuntimeDeps({
installRoot: installParams.installRoot,
installExecutionRoot: installParams.installExecutionRoot,
missingSpecs: installParams.installSpecs ?? installParams.missingSpecs,
env: params.env,
}));
install({ installRoot, installExecutionRoot, missingSpecs, installSpecs });
if (persistRetainedManifest) {
writeRetainedRuntimeDepsManifest(installRoot, installSpecs);
}
storeSourceCheckoutRuntimeDepsCache({ cacheDir, installRoot });
return { installedSpecs: missingSpecs, retainSpecs: installSpecs };
const install =
params.installDeps ??
((installParams) =>
installBundledRuntimeDeps({
installRoot: installParams.installRoot,
installExecutionRoot: installParams.installExecutionRoot,
missingSpecs: installParams.installSpecs ?? installParams.missingSpecs,
env: params.env,
}));
install({ installRoot, installExecutionRoot, missingSpecs, installSpecs });
if (persistRetainedManifest) {
writeRetainedRuntimeDepsManifest(installRoot, installSpecs);
}
storeSourceCheckoutRuntimeDepsCache({ cacheDir, installRoot });
return { installedSpecs: missingSpecs, retainSpecs: installSpecs };
});
}

View File

@@ -0,0 +1,64 @@
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { isLiveTestEnabled } from "../src/agents/live-test-helpers.js";
import { isTruthyEnvValue } from "../src/infra/env.js";
const GOOGLE_IMAGE_KEY =
process.env.GEMINI_API_KEY?.trim() || process.env.GOOGLE_API_KEY?.trim() || "";
const LIVE =
isLiveTestEnabled() &&
isTruthyEnvValue(process.env.OPENCLAW_LIVE_INFER_CLI_TEST) &&
GOOGLE_IMAGE_KEY.length > 0;
const describeLive = LIVE ? describe : describe.skip;
function parseJsonEnvelope(stdout: string): Record<string, unknown> {
const trimmed = stdout.trim();
const jsonStart = trimmed.lastIndexOf("\n{");
const rawJson = jsonStart >= 0 ? trimmed.slice(jsonStart + 1) : trimmed;
return JSON.parse(rawJson) as Record<string, unknown>;
}
describeLive("image generation infer CLI live", () => {
it("generates an image through openclaw infer", () => {
const outputBase = path.join(os.tmpdir(), `openclaw-infer-image-${process.pid}.png`);
const result = spawnSync(
process.execPath,
[
"scripts/run-node.mjs",
"infer",
"image",
"generate",
"--model",
"google/gemini-3.1-flash-image-preview",
"--prompt",
"Minimal flat test image: one blue square on a white background, no text.",
"--output",
outputBase,
"--json",
],
{
cwd: path.resolve(import.meta.dirname, ".."),
encoding: "utf8",
env: process.env,
timeout: 180_000,
},
);
expect(result.status, `${result.stderr}\n${result.stdout}`).toBe(0);
const payload = parseJsonEnvelope(result.stdout);
expect(payload.ok).toBe(true);
expect(payload.capability).toBe("image.generate");
expect(payload.provider).toBe("google");
expect(payload.model).toBe("gemini-3.1-flash-image-preview");
const outputs = payload.outputs as Array<{ path?: string; mimeType?: string; size?: number }>;
expect(outputs).toHaveLength(1);
const outputPath = outputs[0]?.path;
expect(outputPath).toBeTruthy();
expect(fs.existsSync(outputPath ?? "")).toBe(true);
expect(outputs[0]?.mimeType?.startsWith("image/")).toBe(true);
expect(outputs[0]?.size ?? 0).toBeGreaterThan(512);
}, 240_000);
});