mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-25 16:12:13 +00:00
fix(test): speed up openshell remote fs bridge
This commit is contained in:
Vincent Koc
@@ -1,4 +1,4 @@
|
||||
import { execFileSync, spawn } from "node:child_process";
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -8,15 +8,6 @@ import type { OpenShellSandboxBackend } from "./backend.js";
|
||||
import { createOpenShellRemoteFsBridge } from "./remote-fs-bridge.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
const localPythonBin = (() => {
|
||||
try {
|
||||
return execFileSync("python3", ["-c", "import sys; print(sys.executable)"], {
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
} catch {
|
||||
return "python3";
|
||||
}
|
||||
})();
|
||||
|
||||
async function makeTempDir(prefix: string) {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
@@ -46,40 +37,18 @@ async function runLocalShell(params: {
|
||||
roots: { workspace: string; agent: string };
|
||||
}) {
|
||||
const translatedArgs = (params.args ?? []).map((arg) => translateRemotePath(arg, params.roots));
|
||||
const script = normalizeScriptForLocalShell(params.script);
|
||||
const result = await new Promise<{ stdout: Buffer; stderr: Buffer; code: number }>(
|
||||
(resolve, reject) => {
|
||||
const child = spawn("/bin/sh", ["-c", script, "openshell-test", ...translatedArgs], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
const stderrChunks: Buffer[] = [];
|
||||
child.stdout.on("data", (chunk) => stdoutChunks.push(Buffer.from(chunk)));
|
||||
child.stderr.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
const result = {
|
||||
stdout: Buffer.concat(stdoutChunks),
|
||||
stderr: Buffer.concat(stderrChunks),
|
||||
code: code ?? 0,
|
||||
};
|
||||
if (result.code !== 0 && !params.allowFailure) {
|
||||
reject(
|
||||
new Error(
|
||||
result.stderr.toString("utf8").trim() || `script exited with code ${result.code}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
resolve(result);
|
||||
});
|
||||
if (params.stdin !== undefined) {
|
||||
child.stdin.end(params.stdin);
|
||||
return;
|
||||
}
|
||||
child.stdin.end();
|
||||
},
|
||||
);
|
||||
const stdinBuffer =
|
||||
params.stdin === undefined
|
||||
? undefined
|
||||
: Buffer.isBuffer(params.stdin)
|
||||
? params.stdin
|
||||
: Buffer.from(params.stdin);
|
||||
const result = await emulateRemoteShell({
|
||||
script: params.script,
|
||||
args: translatedArgs,
|
||||
stdin: stdinBuffer,
|
||||
allowFailure: params.allowFailure,
|
||||
});
|
||||
return {
|
||||
...result,
|
||||
stdout: Buffer.from(rewriteLocalPaths(result.stdout.toString("utf8"), params.roots), "utf8"),
|
||||
@@ -113,28 +82,146 @@ function rewriteLocalPaths(value: string, roots: { workspace: string; agent: str
|
||||
return value.replaceAll(roots.workspace, "/sandbox").replaceAll(roots.agent, "/agent");
|
||||
}
|
||||
|
||||
function normalizeScriptForLocalShell(script: string) {
|
||||
return script
|
||||
.replace(
|
||||
'stats=$(stat -c "%F|%h" -- "$1")',
|
||||
`stats=$(${localPythonBin} - "$1" <<'PY'
|
||||
import os, stat, sys
|
||||
st = os.stat(sys.argv[1])
|
||||
kind = 'directory' if stat.S_ISDIR(st.st_mode) else 'regular file' if stat.S_ISREG(st.st_mode) else 'other'
|
||||
print(f"{kind}|{st.st_nlink}")
|
||||
PY
|
||||
)`,
|
||||
)
|
||||
.replace(
|
||||
'stat -c "%F|%s|%Y" -- "$1"',
|
||||
`${localPythonBin} - "$1" <<'PY'
|
||||
import os, stat, sys
|
||||
st = os.stat(sys.argv[1])
|
||||
kind = 'directory' if stat.S_ISDIR(st.st_mode) else 'regular file' if stat.S_ISREG(st.st_mode) else 'other'
|
||||
print(f"{kind}|{st.st_size}|{int(st.st_mtime)}")
|
||||
PY`,
|
||||
)
|
||||
.replace("python3 /dev/fd/3", `${localPythonBin} /dev/fd/3`);
|
||||
async function emulateRemoteShell(params: {
|
||||
script: string;
|
||||
args: string[];
|
||||
stdin?: Buffer;
|
||||
allowFailure?: boolean;
|
||||
}): Promise<{ stdout: Buffer; stderr: Buffer; code: number }> {
|
||||
try {
|
||||
if (params.script === 'set -eu\ncat -- "$1"') {
|
||||
return { stdout: await fs.readFile(params.args[0] ?? ""), stderr: Buffer.alloc(0), code: 0 };
|
||||
}
|
||||
|
||||
if (params.script === 'if [ -e "$1" ] || [ -L "$1" ]; then printf "1\\n"; else printf "0\\n"; fi') {
|
||||
const target = params.args[0] ?? "";
|
||||
const exists = await pathExistsOrSymlink(target);
|
||||
return { stdout: Buffer.from(exists ? "1\n" : "0\n"), stderr: Buffer.alloc(0), code: 0 };
|
||||
}
|
||||
|
||||
if (params.script.includes('canonical=$(readlink -f -- "$cursor")')) {
|
||||
const canonical = await resolveCanonicalPath(params.args[0] ?? "", params.args[1] === "1");
|
||||
return { stdout: Buffer.from(`${canonical}\n`), stderr: Buffer.alloc(0), code: 0 };
|
||||
}
|
||||
|
||||
if (params.script.includes('stats=$(stat -c "%F|%h" -- "$1")')) {
|
||||
const target = params.args[0] ?? "";
|
||||
if (!(await pathExistsOrSymlink(target))) {
|
||||
return { stdout: Buffer.alloc(0), stderr: Buffer.alloc(0), code: 0 };
|
||||
}
|
||||
const stats = await fs.lstat(target);
|
||||
return {
|
||||
stdout: Buffer.from(`${describeKind(stats)}|${String(stats.nlink)}\n`),
|
||||
stderr: Buffer.alloc(0),
|
||||
code: 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (params.script.includes('stat -c "%F|%s|%Y" -- "$1"')) {
|
||||
const target = params.args[0] ?? "";
|
||||
const stats = await fs.lstat(target);
|
||||
return {
|
||||
stdout: Buffer.from(
|
||||
`${describeKind(stats)}|${String(stats.size)}|${String(Math.trunc(stats.mtimeMs / 1000))}\n`,
|
||||
),
|
||||
stderr: Buffer.alloc(0),
|
||||
code: 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (params.script.includes('python3 /dev/fd/3 "$@" 3<<\'PY\'')) {
|
||||
await applyMutation(params.args, params.stdin);
|
||||
return { stdout: Buffer.alloc(0), stderr: Buffer.alloc(0), code: 0 };
|
||||
}
|
||||
|
||||
throw new Error(`unsupported remote shell script: ${params.script}`);
|
||||
} catch (error) {
|
||||
if (!params.allowFailure) {
|
||||
throw error;
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return { stdout: Buffer.alloc(0), stderr: Buffer.from(message), code: 1 };
|
||||
}
|
||||
}
|
||||
|
||||
async function pathExistsOrSymlink(target: string) {
|
||||
try {
|
||||
await fs.lstat(target);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function describeKind(stats: fsSync.Stats) {
|
||||
if (stats.isDirectory()) {
|
||||
return "directory";
|
||||
}
|
||||
if (stats.isFile()) {
|
||||
return "regular file";
|
||||
}
|
||||
return "other";
|
||||
}
|
||||
|
||||
async function resolveCanonicalPath(target: string, allowFinalSymlink: boolean) {
|
||||
let suffix = "";
|
||||
let cursor = target;
|
||||
if (allowFinalSymlink && (await isSymlink(target))) {
|
||||
cursor = path.dirname(target);
|
||||
}
|
||||
while (!(await pathExistsOrSymlink(cursor))) {
|
||||
const parent = path.dirname(cursor);
|
||||
if (parent === cursor) {
|
||||
break;
|
||||
}
|
||||
suffix = `${path.posix.sep}${path.basename(cursor)}${suffix}`;
|
||||
cursor = parent;
|
||||
}
|
||||
const canonical = await fs.realpath(cursor);
|
||||
return `${canonical}${suffix}`;
|
||||
}
|
||||
|
||||
async function isSymlink(target: string) {
|
||||
try {
|
||||
return (await fs.lstat(target)).isSymbolicLink();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function applyMutation(args: string[], stdin?: Buffer) {
|
||||
const operation = args[0];
|
||||
if (operation === "write") {
|
||||
const [root, relativeParent, basename, mkdir] = args.slice(1);
|
||||
const parent = path.join(root ?? "", relativeParent ?? "");
|
||||
if (mkdir === "1") {
|
||||
await fs.mkdir(parent, { recursive: true });
|
||||
}
|
||||
await fs.writeFile(path.join(parent, basename ?? ""), stdin ?? Buffer.alloc(0));
|
||||
return;
|
||||
}
|
||||
if (operation === "mkdirp") {
|
||||
const [root, relativePath] = args.slice(1);
|
||||
await fs.mkdir(path.join(root ?? "", relativePath ?? ""), { recursive: true });
|
||||
return;
|
||||
}
|
||||
if (operation === "remove") {
|
||||
const [root, relativeParent, basename, recursive, force] = args.slice(1);
|
||||
const target = path.join(root ?? "", relativeParent ?? "", basename ?? "");
|
||||
await fs.rm(target, { recursive: recursive === "1", force: force !== "0" });
|
||||
return;
|
||||
}
|
||||
if (operation === "rename") {
|
||||
const [srcRoot, srcParent, srcBase, dstRoot, dstParent, dstBase, mkdir] = args.slice(1);
|
||||
const source = path.join(srcRoot ?? "", srcParent ?? "", srcBase ?? "");
|
||||
const destinationParent = path.join(dstRoot ?? "", dstParent ?? "");
|
||||
if (mkdir === "1") {
|
||||
await fs.mkdir(destinationParent, { recursive: true });
|
||||
}
|
||||
await fs.rename(source, path.join(destinationParent, dstBase ?? ""));
|
||||
return;
|
||||
}
|
||||
throw new Error(`unknown mutation operation: ${operation}`);
|
||||
}
|
||||
|
||||
describe("openshell remote fs bridge", () => {
|
||||
|
||||
Reference in New Issue
Block a user