mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 13:50:49 +00:00
test: improve Docker aggregate scheduling
This commit is contained in:
Peter Steinberger
@@ -1043,6 +1043,7 @@ run_update_scenario() {
|
||||
if ! timeout "$DOCKER_RUN_TIMEOUT" docker run --rm \
|
||||
-e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
|
||||
-e OPENCLAW_BUNDLED_CHANNEL_UPDATE_BASELINE_VERSION="$UPDATE_BASELINE_VERSION" \
|
||||
-e "OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=${OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS:-telegram,discord,slack,feishu,memory-lancedb,acpx}" \
|
||||
"${PACKAGE_DOCKER_ARGS[@]}" \
|
||||
-i "$IMAGE_NAME" bash -s >"$run_log" 2>&1 <<'EOF'
|
||||
set -euo pipefail
|
||||
@@ -1056,6 +1057,7 @@ export OPENCLAW_UPDATE_PACKAGE_SPEC=""
|
||||
|
||||
TOKEN="bundled-channel-update-token"
|
||||
PORT="18790"
|
||||
UPDATE_TARGETS="${OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS:-telegram,discord,slack,feishu,memory-lancedb,acpx}"
|
||||
|
||||
package_root() {
|
||||
printf "%s/openclaw" "$(npm root -g)"
|
||||
@@ -1291,7 +1293,16 @@ run_update_and_capture() {
|
||||
fi
|
||||
}
|
||||
|
||||
should_run_update_target() {
|
||||
local target="$1"
|
||||
case ",$UPDATE_TARGETS," in
|
||||
*",all,"* | *",$target,"*) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
echo "Installing current candidate as update baseline..."
|
||||
echo "Update targets: $UPDATE_TARGETS"
|
||||
npm install -g "$package_tgz" --no-fund --no-audit >/tmp/openclaw-update-baseline-install.log 2>&1
|
||||
command -v openclaw >/dev/null
|
||||
baseline_root="$(package_root)"
|
||||
@@ -1299,76 +1310,88 @@ test -d "$baseline_root/dist/extensions/telegram"
|
||||
test -d "$baseline_root/dist/extensions/feishu"
|
||||
test -d "$baseline_root/dist/extensions/acpx"
|
||||
|
||||
echo "Replicating configured Telegram missing-runtime state..."
|
||||
write_config telegram
|
||||
assert_no_dep_available telegram grammy
|
||||
set +e
|
||||
openclaw doctor --non-interactive >/tmp/openclaw-baseline-doctor.log 2>&1
|
||||
baseline_doctor_status=$?
|
||||
set -e
|
||||
echo "baseline doctor exited with $baseline_doctor_status"
|
||||
remove_runtime_dep telegram grammy
|
||||
assert_no_dep_available telegram grammy
|
||||
if should_run_update_target telegram; then
|
||||
echo "Replicating configured Telegram missing-runtime state..."
|
||||
write_config telegram
|
||||
assert_no_dep_available telegram grammy
|
||||
set +e
|
||||
openclaw doctor --non-interactive >/tmp/openclaw-baseline-doctor.log 2>&1
|
||||
baseline_doctor_status=$?
|
||||
set -e
|
||||
echo "baseline doctor exited with $baseline_doctor_status"
|
||||
remove_runtime_dep telegram grammy
|
||||
assert_no_dep_available telegram grammy
|
||||
|
||||
echo "Updating from baseline to current candidate; candidate doctor must repair Telegram deps..."
|
||||
run_update_and_capture telegram /tmp/openclaw-update-telegram.json
|
||||
cat /tmp/openclaw-update-telegram.json
|
||||
assert_update_ok /tmp/openclaw-update-telegram.json "$candidate_version"
|
||||
assert_dep_available telegram grammy
|
||||
echo "Updating from baseline to current candidate; candidate doctor must repair Telegram deps..."
|
||||
run_update_and_capture telegram /tmp/openclaw-update-telegram.json
|
||||
cat /tmp/openclaw-update-telegram.json
|
||||
assert_update_ok /tmp/openclaw-update-telegram.json "$candidate_version"
|
||||
assert_dep_available telegram grammy
|
||||
|
||||
echo "Mutating installed package: remove Telegram deps, then update-mode doctor repairs them..."
|
||||
remove_runtime_dep telegram grammy
|
||||
assert_no_dep_available telegram grammy
|
||||
if ! OPENCLAW_UPDATE_IN_PROGRESS=1 openclaw doctor --non-interactive >/tmp/openclaw-update-mode-doctor.log 2>&1; then
|
||||
echo "update-mode doctor failed while repairing Telegram deps" >&2
|
||||
cat /tmp/openclaw-update-mode-doctor.log >&2
|
||||
exit 1
|
||||
echo "Mutating installed package: remove Telegram deps, then update-mode doctor repairs them..."
|
||||
remove_runtime_dep telegram grammy
|
||||
assert_no_dep_available telegram grammy
|
||||
if ! OPENCLAW_UPDATE_IN_PROGRESS=1 openclaw doctor --non-interactive >/tmp/openclaw-update-mode-doctor.log 2>&1; then
|
||||
echo "update-mode doctor failed while repairing Telegram deps" >&2
|
||||
cat /tmp/openclaw-update-mode-doctor.log >&2
|
||||
exit 1
|
||||
fi
|
||||
assert_dep_available telegram grammy
|
||||
fi
|
||||
assert_dep_available telegram grammy
|
||||
|
||||
echo "Mutating config to Discord and rerunning same-version update path..."
|
||||
write_config discord
|
||||
remove_runtime_dep discord discord-api-types
|
||||
assert_no_dep_available discord discord-api-types
|
||||
run_update_and_capture discord /tmp/openclaw-update-discord.json
|
||||
cat /tmp/openclaw-update-discord.json
|
||||
assert_update_ok /tmp/openclaw-update-discord.json "$candidate_version"
|
||||
assert_dep_available discord discord-api-types
|
||||
if should_run_update_target discord; then
|
||||
echo "Mutating config to Discord and rerunning same-version update path..."
|
||||
write_config discord
|
||||
remove_runtime_dep discord discord-api-types
|
||||
assert_no_dep_available discord discord-api-types
|
||||
run_update_and_capture discord /tmp/openclaw-update-discord.json
|
||||
cat /tmp/openclaw-update-discord.json
|
||||
assert_update_ok /tmp/openclaw-update-discord.json "$candidate_version"
|
||||
assert_dep_available discord discord-api-types
|
||||
fi
|
||||
|
||||
echo "Mutating config to Slack and rerunning same-version update path..."
|
||||
write_config slack
|
||||
remove_runtime_dep slack @slack/web-api
|
||||
assert_no_dep_available slack @slack/web-api
|
||||
run_update_and_capture slack /tmp/openclaw-update-slack.json
|
||||
cat /tmp/openclaw-update-slack.json
|
||||
assert_update_ok /tmp/openclaw-update-slack.json "$candidate_version"
|
||||
assert_dep_available slack @slack/web-api
|
||||
if should_run_update_target slack; then
|
||||
echo "Mutating config to Slack and rerunning same-version update path..."
|
||||
write_config slack
|
||||
remove_runtime_dep slack @slack/web-api
|
||||
assert_no_dep_available slack @slack/web-api
|
||||
run_update_and_capture slack /tmp/openclaw-update-slack.json
|
||||
cat /tmp/openclaw-update-slack.json
|
||||
assert_update_ok /tmp/openclaw-update-slack.json "$candidate_version"
|
||||
assert_dep_available slack @slack/web-api
|
||||
fi
|
||||
|
||||
echo "Mutating config to Feishu and rerunning same-version update path..."
|
||||
write_config feishu
|
||||
remove_runtime_dep feishu @larksuiteoapi/node-sdk
|
||||
assert_no_dep_available feishu @larksuiteoapi/node-sdk
|
||||
run_update_and_capture feishu /tmp/openclaw-update-feishu.json
|
||||
cat /tmp/openclaw-update-feishu.json
|
||||
assert_update_ok /tmp/openclaw-update-feishu.json "$candidate_version"
|
||||
assert_dep_available feishu @larksuiteoapi/node-sdk
|
||||
if should_run_update_target feishu; then
|
||||
echo "Mutating config to Feishu and rerunning same-version update path..."
|
||||
write_config feishu
|
||||
remove_runtime_dep feishu @larksuiteoapi/node-sdk
|
||||
assert_no_dep_available feishu @larksuiteoapi/node-sdk
|
||||
run_update_and_capture feishu /tmp/openclaw-update-feishu.json
|
||||
cat /tmp/openclaw-update-feishu.json
|
||||
assert_update_ok /tmp/openclaw-update-feishu.json "$candidate_version"
|
||||
assert_dep_available feishu @larksuiteoapi/node-sdk
|
||||
fi
|
||||
|
||||
echo "Mutating config to memory-lancedb and rerunning same-version update path..."
|
||||
write_config memory-lancedb
|
||||
remove_runtime_dep memory-lancedb @lancedb/lancedb
|
||||
assert_no_dep_available memory-lancedb @lancedb/lancedb
|
||||
run_update_and_capture memory-lancedb /tmp/openclaw-update-memory-lancedb.json
|
||||
cat /tmp/openclaw-update-memory-lancedb.json
|
||||
assert_update_ok /tmp/openclaw-update-memory-lancedb.json "$candidate_version"
|
||||
assert_dep_available memory-lancedb @lancedb/lancedb
|
||||
if should_run_update_target memory-lancedb; then
|
||||
echo "Mutating config to memory-lancedb and rerunning same-version update path..."
|
||||
write_config memory-lancedb
|
||||
remove_runtime_dep memory-lancedb @lancedb/lancedb
|
||||
assert_no_dep_available memory-lancedb @lancedb/lancedb
|
||||
run_update_and_capture memory-lancedb /tmp/openclaw-update-memory-lancedb.json
|
||||
cat /tmp/openclaw-update-memory-lancedb.json
|
||||
assert_update_ok /tmp/openclaw-update-memory-lancedb.json "$candidate_version"
|
||||
assert_dep_available memory-lancedb @lancedb/lancedb
|
||||
fi
|
||||
|
||||
echo "Removing ACPX runtime package and rerunning same-version update path..."
|
||||
remove_runtime_dep acpx acpx
|
||||
assert_no_dep_available acpx acpx
|
||||
run_update_and_capture acpx /tmp/openclaw-update-acpx.json
|
||||
cat /tmp/openclaw-update-acpx.json
|
||||
assert_update_ok /tmp/openclaw-update-acpx.json "$candidate_version"
|
||||
assert_dep_available acpx acpx
|
||||
if should_run_update_target acpx; then
|
||||
echo "Removing ACPX runtime package and rerunning same-version update path..."
|
||||
remove_runtime_dep acpx acpx
|
||||
assert_no_dep_available acpx acpx
|
||||
run_update_and_capture acpx /tmp/openclaw-update-acpx.json
|
||||
cat /tmp/openclaw-update-acpx.json
|
||||
assert_update_ok /tmp/openclaw-update-acpx.json "$candidate_version"
|
||||
assert_dep_available acpx acpx
|
||||
fi
|
||||
|
||||
echo "bundled channel runtime deps Docker update E2E passed"
|
||||
EOF
|
||||
|
||||
@@ -11,6 +11,14 @@ const DEFAULT_TAIL_PARALLELISM = 10;
|
||||
const DEFAULT_FAILURE_TAIL_LINES = 80;
|
||||
const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000;
|
||||
const DEFAULT_LANE_START_STAGGER_MS = 2_000;
|
||||
const DEFAULT_STATUS_INTERVAL_MS = 30_000;
|
||||
const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000;
|
||||
const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
|
||||
const LIVE_PROFILE_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
const LIVE_CLI_TIMEOUT_MS = 12 * 60 * 1000;
|
||||
const LIVE_ACP_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
const OPENWEBUI_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
const BUNDLED_UPDATE_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
const DEFAULT_RESOURCE_LIMITS = {
|
||||
docker: DEFAULT_PARALLELISM,
|
||||
live: 4,
|
||||
@@ -24,14 +32,17 @@ const bundledChannelLaneCommand =
|
||||
function lane(name, command, options = {}) {
|
||||
return {
|
||||
command,
|
||||
estimateSeconds: options.estimateSeconds,
|
||||
name,
|
||||
resources: options.resources ?? [],
|
||||
timeoutMs: options.timeoutMs,
|
||||
weight: options.weight ?? 1,
|
||||
};
|
||||
}
|
||||
|
||||
function liveLane(name, command, options = {}) {
|
||||
return lane(name, command, {
|
||||
...options,
|
||||
resources: ["live", ...(options.resources ?? [])],
|
||||
weight: options.weight ?? 3,
|
||||
});
|
||||
@@ -39,6 +50,7 @@ function liveLane(name, command, options = {}) {
|
||||
|
||||
function npmLane(name, command, options = {}) {
|
||||
return lane(name, command, {
|
||||
...options,
|
||||
resources: ["npm", ...(options.resources ?? [])],
|
||||
weight: options.weight ?? 2,
|
||||
});
|
||||
@@ -46,6 +58,7 @@ function npmLane(name, command, options = {}) {
|
||||
|
||||
function serviceLane(name, command, options = {}) {
|
||||
return lane(name, command, {
|
||||
...options,
|
||||
resources: ["service", ...(options.resources ?? [])],
|
||||
weight: options.weight ?? 2,
|
||||
});
|
||||
@@ -70,8 +83,34 @@ const bundledScenarioLanes = [
|
||||
`OPENCLAW_BUNDLED_CHANNELS=memory-lancedb ${bundledChannelLaneCommand}`,
|
||||
),
|
||||
npmLane(
|
||||
"bundled-channel-update",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
||||
"bundled-channel-update-telegram",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=telegram OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
||||
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
||||
),
|
||||
npmLane(
|
||||
"bundled-channel-update-discord",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=discord OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
||||
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
||||
),
|
||||
npmLane(
|
||||
"bundled-channel-update-slack",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=slack OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
||||
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
||||
),
|
||||
npmLane(
|
||||
"bundled-channel-update-feishu",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=feishu OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
||||
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
||||
),
|
||||
npmLane(
|
||||
"bundled-channel-update-memory-lancedb",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=memory-lancedb OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
||||
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
||||
),
|
||||
npmLane(
|
||||
"bundled-channel-update-acpx",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=acpx OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
||||
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
||||
),
|
||||
npmLane(
|
||||
"bundled-channel-root-owned",
|
||||
@@ -93,22 +132,25 @@ const bundledScenarioLanes = [
|
||||
|
||||
const lanes = [
|
||||
liveLane("live-models", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-models", {
|
||||
timeoutMs: LIVE_PROFILE_TIMEOUT_MS,
|
||||
weight: 4,
|
||||
}),
|
||||
liveLane("live-gateway", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-gateway", {
|
||||
timeoutMs: LIVE_PROFILE_TIMEOUT_MS,
|
||||
weight: 4,
|
||||
}),
|
||||
liveLane(
|
||||
"live-cli-backend-claude",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:claude",
|
||||
{ resources: ["npm"], weight: 3 },
|
||||
{ resources: ["npm"], timeoutMs: LIVE_CLI_TIMEOUT_MS, weight: 3 },
|
||||
),
|
||||
liveLane(
|
||||
"live-cli-backend-gemini",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:gemini",
|
||||
{ resources: ["npm"], weight: 3 },
|
||||
{ resources: ["npm"], timeoutMs: LIVE_CLI_TIMEOUT_MS, weight: 3 },
|
||||
),
|
||||
serviceLane("openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui", {
|
||||
timeoutMs: OPENWEBUI_TIMEOUT_MS,
|
||||
weight: 5,
|
||||
}),
|
||||
serviceLane("onboard", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:onboard", {
|
||||
@@ -145,35 +187,37 @@ const exclusiveLanes = [
|
||||
serviceLane(
|
||||
"openai-web-search-minimal",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-web-search-minimal",
|
||||
{ timeoutMs: 8 * 60 * 1000 },
|
||||
),
|
||||
liveLane(
|
||||
"live-codex-harness",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-codex-harness",
|
||||
{ resources: ["npm"], weight: 3 },
|
||||
{ resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3 },
|
||||
),
|
||||
liveLane("live-codex-bind", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-codex-bind", {
|
||||
resources: ["npm"],
|
||||
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
||||
weight: 3,
|
||||
}),
|
||||
liveLane(
|
||||
"live-cli-backend-codex",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:codex",
|
||||
{ resources: ["npm"], weight: 3 },
|
||||
{ resources: ["npm"], timeoutMs: LIVE_CLI_TIMEOUT_MS, weight: 3 },
|
||||
),
|
||||
liveLane(
|
||||
"live-acp-bind-claude",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:claude",
|
||||
{ resources: ["npm"], weight: 3 },
|
||||
{ resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3 },
|
||||
),
|
||||
liveLane(
|
||||
"live-acp-bind-codex",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:codex",
|
||||
{ resources: ["npm"], weight: 3 },
|
||||
{ resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3 },
|
||||
),
|
||||
liveLane(
|
||||
"live-acp-bind-gemini",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:gemini",
|
||||
{ resources: ["npm"], weight: 3 },
|
||||
{ resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3 },
|
||||
),
|
||||
];
|
||||
|
||||
@@ -240,7 +284,8 @@ function laneResources(poolLane) {
|
||||
|
||||
function laneSummary(poolLane) {
|
||||
const resources = laneResources(poolLane).join(",");
|
||||
return `${poolLane.name}(w=${laneWeight(poolLane)} r=${resources})`;
|
||||
const timeout = poolLane.timeoutMs ? ` timeout=${Math.round(poolLane.timeoutMs / 1000)}s` : "";
|
||||
return `${poolLane.name}(w=${laneWeight(poolLane)} r=${resources}${timeout})`;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
@@ -277,6 +322,91 @@ function shellQuote(value) {
|
||||
return `'${String(value).replaceAll("'", "'\\''")}'`;
|
||||
}
|
||||
|
||||
function timingSeconds(timingStore, poolLane) {
|
||||
const fromStore = timingStore?.lanes?.[poolLane.name]?.durationSeconds;
|
||||
if (typeof fromStore === "number" && Number.isFinite(fromStore) && fromStore > 0) {
|
||||
return fromStore;
|
||||
}
|
||||
return poolLane.estimateSeconds ?? 0;
|
||||
}
|
||||
|
||||
function orderLanes(poolLanes, timingStore) {
|
||||
return poolLanes
|
||||
.map((poolLane, index) => ({ index, poolLane, seconds: timingSeconds(timingStore, poolLane) }))
|
||||
.toSorted((a, b) => b.seconds - a.seconds || a.index - b.index)
|
||||
.map(({ poolLane }) => poolLane);
|
||||
}
|
||||
|
||||
async function loadTimingStore(file, enabled) {
|
||||
if (!enabled) {
|
||||
return { enabled: false, file, lanes: {}, version: 1 };
|
||||
}
|
||||
const raw = await readFile(file, "utf8").catch(() => "");
|
||||
if (!raw.trim()) {
|
||||
return { enabled: true, file, lanes: {}, version: 1 };
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return {
|
||||
enabled: true,
|
||||
file,
|
||||
lanes: parsed && typeof parsed.lanes === "object" && parsed.lanes ? parsed.lanes : {},
|
||||
version: 1,
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`WARN: ignoring unreadable Docker lane timings ${file}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
return { enabled: true, file, lanes: {}, version: 1 };
|
||||
}
|
||||
}
|
||||
|
||||
async function writeTimingStore(timingStore, results) {
|
||||
if (!timingStore.enabled || results.length === 0) {
|
||||
return;
|
||||
}
|
||||
const next = {
|
||||
lanes: { ...timingStore.lanes },
|
||||
updatedAt: new Date().toISOString(),
|
||||
version: 1,
|
||||
};
|
||||
for (const result of results) {
|
||||
if (!result || typeof result.elapsedSeconds !== "number") {
|
||||
continue;
|
||||
}
|
||||
next.lanes[result.name] = {
|
||||
durationSeconds: result.elapsedSeconds,
|
||||
status: result.status,
|
||||
timedOut: result.timedOut,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
await mkdir(path.dirname(timingStore.file), { recursive: true });
|
||||
await fs.promises.writeFile(timingStore.file, `${JSON.stringify(next, null, 2)}\n`);
|
||||
timingStore.lanes = next.lanes;
|
||||
console.log(`==> Docker lane timings: ${timingStore.file}`);
|
||||
}
|
||||
|
||||
function printLaneManifest(label, poolLanes, timingStore) {
|
||||
console.log(`==> ${label} lanes (${poolLanes.length})`);
|
||||
for (const [index, poolLane] of poolLanes.entries()) {
|
||||
const seconds = timingSeconds(timingStore, poolLane);
|
||||
const estimate = seconds > 0 ? ` last=${Math.round(seconds)}s` : "";
|
||||
console.log(` ${index + 1}. ${laneSummary(poolLane)}${estimate}`);
|
||||
}
|
||||
}
|
||||
|
||||
function dockerPreflightContainerNames(raw) {
|
||||
return raw
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim().split(/\s+/, 1)[0])
|
||||
.filter((name) =>
|
||||
/^(?:openclaw-(?:gateway-e2e|openwebui|openwebui-gateway|config-reload-e2e)-)/.test(name),
|
||||
);
|
||||
}
|
||||
|
||||
function runShellCommand({ command, env, label, logFile, timeoutMs }) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("bash", ["-lc", command], {
|
||||
@@ -329,6 +459,44 @@ function runShellCommand({ command, env, label, logFile, timeoutMs }) {
|
||||
});
|
||||
}
|
||||
|
||||
function runShellCaptureCommand({ command, env, label, timeoutMs }) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("bash", ["-lc", command], {
|
||||
cwd: ROOT_DIR,
|
||||
detached: process.platform !== "win32",
|
||||
env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
activeChildren.add(child);
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
const timeoutTimer =
|
||||
timeoutMs > 0
|
||||
? setTimeout(() => {
|
||||
timedOut = true;
|
||||
terminateChild(child, "SIGTERM");
|
||||
setTimeout(() => terminateChild(child, "SIGKILL"), 10_000).unref?.();
|
||||
}, timeoutMs)
|
||||
: undefined;
|
||||
timeoutTimer?.unref?.();
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += String(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
if (timeoutTimer) {
|
||||
clearTimeout(timeoutTimer);
|
||||
}
|
||||
activeChildren.delete(child);
|
||||
const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
|
||||
resolve({ label, signal, status: exitCode, stderr, stdout, timedOut });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function runForeground(label, command, env) {
|
||||
console.log(`==> ${label}`);
|
||||
const result = await runShellCommand({ command, env, label });
|
||||
@@ -357,6 +525,66 @@ async function runForegroundGroup(entries, env) {
|
||||
}
|
||||
}
|
||||
|
||||
async function runDockerPreflight(baseEnv, options) {
|
||||
if (!options.enabled) {
|
||||
console.log("==> Docker preflight: skipped");
|
||||
return;
|
||||
}
|
||||
console.log("==> Docker preflight");
|
||||
const version = await runShellCaptureCommand({
|
||||
command: "docker version --format '{{.Server.Version}}'",
|
||||
env: baseEnv,
|
||||
label: "docker-version",
|
||||
timeoutMs: 20_000,
|
||||
});
|
||||
if (version.status !== 0) {
|
||||
throw new Error(
|
||||
`Docker preflight failed: docker version status=${version.status}\n${version.stderr}${version.stdout}`,
|
||||
);
|
||||
}
|
||||
console.log(`==> Docker server: ${version.stdout.trim()}`);
|
||||
|
||||
if (options.cleanup) {
|
||||
const stale = await runShellCaptureCommand({
|
||||
command:
|
||||
"docker ps -a --filter status=created --filter status=exited --filter status=dead --format '{{.Names}} {{.Status}}'",
|
||||
env: baseEnv,
|
||||
label: "docker-stale-list",
|
||||
timeoutMs: 20_000,
|
||||
});
|
||||
if (stale.status === 0) {
|
||||
const names = dockerPreflightContainerNames(stale.stdout);
|
||||
if (names.length > 0) {
|
||||
console.log(`==> Docker preflight cleanup: ${names.join(", ")}`);
|
||||
const cleanup = await runShellCommand({
|
||||
command: `docker rm -f ${names.map(shellQuote).join(" ")}`,
|
||||
env: baseEnv,
|
||||
label: "docker-stale-cleanup",
|
||||
timeoutMs: 90_000,
|
||||
});
|
||||
if (cleanup.status !== 0) {
|
||||
throw new Error(`Docker preflight cleanup failed with status ${cleanup.status}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
const run = await runShellCommand({
|
||||
command: "docker run --rm alpine:3.20 true",
|
||||
env: baseEnv,
|
||||
label: "docker-run-smoke",
|
||||
timeoutMs: options.runTimeoutMs,
|
||||
});
|
||||
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
|
||||
if (run.status !== 0) {
|
||||
throw new Error(
|
||||
`Docker preflight failed: docker run alpine:3.20 true status=${run.status} elapsed=${elapsedSeconds}s`,
|
||||
);
|
||||
}
|
||||
console.log(`==> Docker preflight run: ${elapsedSeconds}s`);
|
||||
}
|
||||
|
||||
async function prepareBundledChannelPackage(baseEnv, logDir) {
|
||||
if (baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ) {
|
||||
console.log(`==> Bundled channel package: ${baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ}`);
|
||||
@@ -411,8 +639,9 @@ function laneEnv(name, baseEnv, logDir) {
|
||||
return env;
|
||||
}
|
||||
|
||||
async function runLane(lane, baseEnv, logDir, timeoutMs) {
|
||||
async function runLane(lane, baseEnv, logDir, fallbackTimeoutMs) {
|
||||
const { command, name } = lane;
|
||||
const timeoutMs = lane.timeoutMs ?? fallbackTimeoutMs;
|
||||
const logFile = path.join(logDir, `${name}.log`);
|
||||
const env = laneEnv(name, baseEnv, logDir);
|
||||
await mkdir(env.OPENCLAW_DOCKER_CLI_TOOLS_DIR, { recursive: true });
|
||||
@@ -422,6 +651,7 @@ async function runLane(lane, baseEnv, logDir, timeoutMs) {
|
||||
[
|
||||
`==> [${name}] cli tools dir: ${env.OPENCLAW_DOCKER_CLI_TOOLS_DIR}`,
|
||||
`==> [${name}] cache dir: ${env.OPENCLAW_DOCKER_CACHE_HOME_DIR}`,
|
||||
`==> [${name}] timeout: ${timeoutMs}ms`,
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
@@ -441,6 +671,7 @@ async function runLane(lane, baseEnv, logDir, timeoutMs) {
|
||||
command,
|
||||
logFile,
|
||||
name,
|
||||
elapsedSeconds,
|
||||
status: result.status,
|
||||
timedOut: result.timedOut,
|
||||
};
|
||||
@@ -448,6 +679,7 @@ async function runLane(lane, baseEnv, logDir, timeoutMs) {
|
||||
|
||||
async function runLanePool(poolLanes, baseEnv, logDir, parallelism, options) {
|
||||
const failures = [];
|
||||
const results = [];
|
||||
const pending = [...poolLanes];
|
||||
const running = new Set();
|
||||
const active = {
|
||||
@@ -455,8 +687,26 @@ async function runLanePool(poolLanes, baseEnv, logDir, parallelism, options) {
|
||||
resources: new Map(),
|
||||
weight: 0,
|
||||
};
|
||||
const activeLanes = new Map();
|
||||
let lastLaneStartAt = 0;
|
||||
let laneStartQueue = Promise.resolve();
|
||||
const statusTimer =
|
||||
options.statusIntervalMs > 0
|
||||
? setInterval(() => {
|
||||
const runningSummary = [...activeLanes.values()]
|
||||
.map((entry) => `${entry.name}:${Math.round((Date.now() - entry.startedAt) / 1000)}s`)
|
||||
.join(", ");
|
||||
const resources = [...active.resources.entries()]
|
||||
.map(([resource, value]) => `${resource}=${value}`)
|
||||
.join(" ");
|
||||
console.log(
|
||||
`==> [${options.poolLabel}] active=${active.count} pending=${pending.length} ${resources}${
|
||||
runningSummary ? ` lanes=${runningSummary}` : ""
|
||||
}`,
|
||||
);
|
||||
}, options.statusIntervalMs)
|
||||
: undefined;
|
||||
statusTimer?.unref?.();
|
||||
|
||||
async function waitForLaneStartSlot() {
|
||||
if (options.startStaggerMs <= 0) {
|
||||
@@ -517,56 +767,66 @@ async function runLanePool(poolLanes, baseEnv, logDir, parallelism, options) {
|
||||
async function startLane(poolLane) {
|
||||
await waitForLaneStartSlot();
|
||||
reserve(poolLane);
|
||||
activeLanes.set(poolLane.name, { name: poolLane.name, startedAt: Date.now() });
|
||||
let promise;
|
||||
promise = runLane(poolLane, baseEnv, logDir, options.timeoutMs)
|
||||
.then((result) => ({ lane: poolLane, promise, result }))
|
||||
.finally(() => {
|
||||
activeLanes.delete(poolLane.name);
|
||||
release(poolLane);
|
||||
});
|
||||
running.add(promise);
|
||||
}
|
||||
|
||||
while (pending.length > 0 || running.size > 0) {
|
||||
let started = false;
|
||||
if (!options.failFast || failures.length === 0) {
|
||||
for (let index = 0; index < pending.length; ) {
|
||||
const candidate = pending[index];
|
||||
if (!canStartLane(candidate)) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
pending.splice(index, 1);
|
||||
await startLane(candidate);
|
||||
started = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (started) {
|
||||
continue;
|
||||
}
|
||||
if (running.size === 0) {
|
||||
const blocked = pending.map(laneSummary).join(", ");
|
||||
throw new Error(`No Docker lanes fit scheduler limits: ${blocked}`);
|
||||
}
|
||||
|
||||
const { promise, result } = await Promise.race(running);
|
||||
running.delete(promise);
|
||||
if (result.status !== 0) {
|
||||
failures.push(result);
|
||||
}
|
||||
if (options.failFast && failures.length > 0) {
|
||||
const remainingResults = await Promise.all(running);
|
||||
running.clear();
|
||||
for (const remaining of remainingResults) {
|
||||
if (remaining.result.status !== 0) {
|
||||
failures.push(remaining.result);
|
||||
try {
|
||||
while (pending.length > 0 || running.size > 0) {
|
||||
let started = false;
|
||||
if (!options.failFast || failures.length === 0) {
|
||||
for (let index = 0; index < pending.length; ) {
|
||||
const candidate = pending[index];
|
||||
if (!canStartLane(candidate)) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
pending.splice(index, 1);
|
||||
await startLane(candidate);
|
||||
started = true;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
if (started) {
|
||||
continue;
|
||||
}
|
||||
if (running.size === 0) {
|
||||
const blocked = pending.map(laneSummary).join(", ");
|
||||
throw new Error(`No Docker lanes fit scheduler limits: ${blocked}`);
|
||||
}
|
||||
|
||||
const { promise, result } = await Promise.race(running);
|
||||
running.delete(promise);
|
||||
results.push(result);
|
||||
if (result.status !== 0) {
|
||||
failures.push(result);
|
||||
}
|
||||
if (options.failFast && failures.length > 0) {
|
||||
const remainingResults = await Promise.all(running);
|
||||
running.clear();
|
||||
for (const remaining of remainingResults) {
|
||||
results.push(remaining.result);
|
||||
if (remaining.result.status !== 0) {
|
||||
failures.push(remaining.result);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (statusTimer) {
|
||||
clearInterval(statusTimer);
|
||||
}
|
||||
}
|
||||
|
||||
return failures;
|
||||
return { failures, results };
|
||||
}
|
||||
|
||||
async function tailFile(file, lines) {
|
||||
@@ -640,7 +900,24 @@ async function main() {
|
||||
DEFAULT_LANE_START_STAGGER_MS,
|
||||
"OPENCLAW_DOCKER_ALL_START_STAGGER_MS",
|
||||
);
|
||||
const statusIntervalMs = parseNonNegativeInt(
|
||||
process.env.OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS,
|
||||
DEFAULT_STATUS_INTERVAL_MS,
|
||||
"OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS",
|
||||
);
|
||||
const preflightRunTimeoutMs = parsePositiveInt(
|
||||
process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS,
|
||||
DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS,
|
||||
"OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS",
|
||||
);
|
||||
const failFast = parseBool(process.env.OPENCLAW_DOCKER_ALL_FAIL_FAST, true);
|
||||
const dryRun = parseBool(process.env.OPENCLAW_DOCKER_ALL_DRY_RUN, false);
|
||||
const preflightEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT, true);
|
||||
const preflightCleanup = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_CLEANUP, true);
|
||||
const timingsEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_TIMINGS, true);
|
||||
const timingsFile = path.resolve(
|
||||
process.env.OPENCLAW_DOCKER_ALL_TIMINGS_FILE || DEFAULT_TIMINGS_FILE,
|
||||
);
|
||||
const runId = process.env.OPENCLAW_DOCKER_ALL_RUN_ID || utcStampForPath();
|
||||
const logDir = path.resolve(
|
||||
process.env.OPENCLAW_DOCKER_ALL_LOG_DIR ||
|
||||
@@ -655,12 +932,24 @@ async function main() {
|
||||
appendExtension(baseEnv, "acpx");
|
||||
appendExtension(baseEnv, "codex");
|
||||
|
||||
const timingStore = await loadTimingStore(timingsFile, timingsEnabled);
|
||||
const orderedLanes = orderLanes(lanes, timingStore);
|
||||
const orderedTailLanes = orderLanes(tailLanes, timingStore);
|
||||
|
||||
console.log(`==> Docker test logs: ${logDir}`);
|
||||
console.log(`==> Parallelism: ${parallelism}`);
|
||||
console.log(`==> Tail parallelism: ${tailParallelism}`);
|
||||
console.log(`==> Lane timeout: ${laneTimeoutMs}ms`);
|
||||
console.log(`==> Lane start stagger: ${laneStartStaggerMs}ms`);
|
||||
console.log(`==> Status interval: ${statusIntervalMs}ms`);
|
||||
console.log(`==> Fail fast: ${failFast ? "yes" : "no"}`);
|
||||
console.log(`==> Dry run: ${dryRun ? "yes" : "no"}`);
|
||||
console.log(
|
||||
`==> Docker preflight: ${preflightEnabled ? "yes" : "no"}${
|
||||
preflightCleanup ? " cleanup=yes" : " cleanup=no"
|
||||
}`,
|
||||
);
|
||||
console.log(`==> Docker lane timings: ${timingStore.enabled ? timingsFile : "disabled"}`);
|
||||
console.log(`==> Live-test bundled plugin deps: ${baseEnv.OPENCLAW_DOCKER_BUILD_EXTENSIONS}`);
|
||||
const schedulerOptions = parseSchedulerOptions(process.env, parallelism);
|
||||
const tailSchedulerOptions = parseSchedulerOptions(process.env, tailParallelism);
|
||||
@@ -670,6 +959,18 @@ async function main() {
|
||||
console.log(
|
||||
`==> Tail scheduler: weight=${tailSchedulerOptions.weightLimit} docker=${tailSchedulerOptions.resourceLimits.docker} live=${tailSchedulerOptions.resourceLimits.live} npm=${tailSchedulerOptions.resourceLimits.npm} service=${tailSchedulerOptions.resourceLimits.service}`,
|
||||
);
|
||||
printLaneManifest("Main", orderedLanes, timingStore);
|
||||
printLaneManifest("Tail", orderedTailLanes, timingStore);
|
||||
if (dryRun) {
|
||||
console.log("==> Dry run complete");
|
||||
return;
|
||||
}
|
||||
|
||||
await runDockerPreflight(baseEnv, {
|
||||
cleanup: preflightCleanup,
|
||||
enabled: preflightEnabled,
|
||||
runTimeoutMs: preflightRunTimeoutMs,
|
||||
});
|
||||
|
||||
await runForegroundGroup(
|
||||
[
|
||||
@@ -686,22 +987,29 @@ async function main() {
|
||||
const options = {
|
||||
...schedulerOptions,
|
||||
failFast,
|
||||
poolLabel: "main",
|
||||
startStaggerMs: laneStartStaggerMs,
|
||||
statusIntervalMs,
|
||||
timeoutMs: laneTimeoutMs,
|
||||
};
|
||||
const failures = await runLanePool(lanes, baseEnv, logDir, parallelism, options);
|
||||
const mainResult = await runLanePool(orderedLanes, baseEnv, logDir, parallelism, options);
|
||||
const failures = [...mainResult.failures];
|
||||
const allResults = [...mainResult.results];
|
||||
await writeTimingStore(timingStore, mainResult.results);
|
||||
if (failFast && failures.length > 0) {
|
||||
await printFailureSummary(failures, tailLines);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("==> Running provider-sensitive Docker tail lanes");
|
||||
failures.push(
|
||||
...(await runLanePool(tailLanes, baseEnv, logDir, tailParallelism, {
|
||||
...options,
|
||||
...tailSchedulerOptions,
|
||||
})),
|
||||
);
|
||||
const tailResult = await runLanePool(orderedTailLanes, baseEnv, logDir, tailParallelism, {
|
||||
...options,
|
||||
...tailSchedulerOptions,
|
||||
poolLabel: "tail",
|
||||
});
|
||||
failures.push(...tailResult.failures);
|
||||
allResults.push(...tailResult.results);
|
||||
await writeTimingStore(timingStore, tailResult.results);
|
||||
if (failures.length > 0) {
|
||||
await printFailureSummary(failures, tailLines);
|
||||
process.exit(1);
|
||||
@@ -712,6 +1020,7 @@ async function main() {
|
||||
"pnpm test:docker:cleanup",
|
||||
baseEnv,
|
||||
);
|
||||
await writeTimingStore(timingStore, allResults);
|
||||
console.log("==> Docker test suite passed");
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user