test: bound Docker smoke host commands

2026-05-06 14:20:44 +00:00 · 2026-04-24 20:54:02 +01:00
parent 0e23107ffb
commit 037d12974c
3 changed files with 68 additions and 43 deletions
--- a/scripts/e2e/gateway-network-docker.sh
+++ b/scripts/e2e/gateway-network-docker.sh
@@ -10,20 +10,26 @@ PORT="18789"
 TOKEN="e2e-$(date +%s)-$$"
 NET_NAME="openclaw-net-e2e-$$"
 GW_NAME="openclaw-gateway-e2e-$$"
+DOCKER_COMMAND_TIMEOUT="${OPENCLAW_GATEWAY_NETWORK_DOCKER_COMMAND_TIMEOUT:-60s}"
+CLIENT_TIMEOUT="${OPENCLAW_GATEWAY_NETWORK_CLIENT_TIMEOUT:-90s}"
+
+docker_cmd() {
+  timeout "$DOCKER_COMMAND_TIMEOUT" "$@"
+}

 cleanup() {
-  docker rm -f "$GW_NAME" >/dev/null 2>&1 || true
-  docker network rm "$NET_NAME" >/dev/null 2>&1 || true
+  docker_cmd docker rm -f "$GW_NAME" >/dev/null 2>&1 || true
+  docker_cmd docker network rm "$NET_NAME" >/dev/null 2>&1 || true
 }
 trap cleanup EXIT

 docker_e2e_build_or_reuse "$IMAGE_NAME" gateway-network "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "" "$SKIP_BUILD"

 echo "Creating Docker network..."
-docker network create "$NET_NAME" >/dev/null
+docker_cmd docker network create "$NET_NAME" >/dev/null

 echo "Starting gateway container..."
-docker run -d \
+docker_cmd docker run -d \
  --name "$GW_NAME" \
  --network "$NET_NAME" \
  -e "OPENCLAW_GATEWAY_TOKEN=$TOKEN" \
@@ -37,10 +43,10 @@ docker run -d \
 echo "Waiting for gateway to come up..."
 ready=0
 for _ in $(seq 1 180); do
-  if [ "$(docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" != "true" ]; then
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" != "true" ]; then
    break
  fi
-  if docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
+  if docker_cmd docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
    import net from \"node:net\";
    const socket = net.createConnection({ host: \"127.0.0.1\", port: $PORT });
    const timeout = setTimeout(() => {
@@ -60,7 +66,7 @@ for _ in $(seq 1 180); do
    ready=1
    break
  fi
-  if docker exec "$GW_NAME" bash -lc "grep -q \"listening on ws://\" /tmp/gateway-net-e2e.log 2>/dev/null"; then
+  if docker_cmd docker exec "$GW_NAME" bash -lc "grep -q \"listening on ws://\" /tmp/gateway-net-e2e.log 2>/dev/null"; then
    ready=1
    break
  fi
@@ -69,16 +75,16 @@ done

 if [ "$ready" -ne 1 ]; then
  echo "Gateway failed to start"
-  if [ "$(docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
-    docker exec "$GW_NAME" bash -lc "tail -n 80 /tmp/gateway-net-e2e.log" || true
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
+    docker_cmd docker exec "$GW_NAME" bash -lc "tail -n 80 /tmp/gateway-net-e2e.log" || true
  else
-    docker logs "$GW_NAME" 2>&1 | tail -n 120 || true
+    docker_cmd docker logs "$GW_NAME" 2>&1 | tail -n 120 || true
  fi
  exit 1
 fi

 echo "Running client container (connect + health)..."
-run_logged gateway-network-client docker run --rm \
+run_logged gateway-network-client timeout "$CLIENT_TIMEOUT" docker run --rm \
  --network "$NET_NAME" \
  -e "GW_URL=ws://$GW_NAME:$PORT" \
  -e "GW_TOKEN=$TOKEN" \
--- a/scripts/e2e/openwebui-docker.sh
+++ b/scripts/e2e/openwebui-docker.sh
@@ -19,6 +19,12 @@ ADMIN_PASSWORD="${OPENCLAW_OPENWEBUI_ADMIN_PASSWORD:-OpenWebUI-E2E-Password-$(da
 NET_NAME="openclaw-openwebui-e2e-$$"
 GW_NAME="openclaw-openwebui-gateway-$$"
 OW_NAME="openclaw-openwebui-$$"
+DOCKER_COMMAND_TIMEOUT="${OPENCLAW_OPENWEBUI_DOCKER_COMMAND_TIMEOUT:-60s}"
+DOCKER_PULL_TIMEOUT="${OPENCLAW_OPENWEBUI_DOCKER_PULL_TIMEOUT:-300s}"
+
+docker_cmd() {
+  timeout "$DOCKER_COMMAND_TIMEOUT" "$@"
+}

 OPENAI_API_KEY_VALUE="${OPENAI_API_KEY:-}"
 if [[ "$OPENAI_API_KEY_VALUE" == "undefined" || "$OPENAI_API_KEY_VALUE" == "null" ]]; then
@@ -34,22 +40,22 @@ if [[ -z "$OPENAI_API_KEY_VALUE" ]]; then
 fi

 cleanup() {
-  docker rm -f "$OW_NAME" >/dev/null 2>&1 || true
-  docker rm -f "$GW_NAME" >/dev/null 2>&1 || true
-  docker network rm "$NET_NAME" >/dev/null 2>&1 || true
+  docker_cmd docker rm -f "$OW_NAME" >/dev/null 2>&1 || true
+  docker_cmd docker rm -f "$GW_NAME" >/dev/null 2>&1 || true
+  docker_cmd docker network rm "$NET_NAME" >/dev/null 2>&1 || true
 }
 trap cleanup EXIT

 docker_e2e_build_or_reuse "$IMAGE_NAME" openwebui

 echo "Pulling Open WebUI image: $OPENWEBUI_IMAGE"
-docker pull "$OPENWEBUI_IMAGE" >/dev/null
+timeout "$DOCKER_PULL_TIMEOUT" docker pull "$OPENWEBUI_IMAGE" >/dev/null

 echo "Creating Docker network..."
-docker network create "$NET_NAME" >/dev/null
+docker_cmd docker network create "$NET_NAME" >/dev/null

 echo "Starting gateway container..."
-docker run -d \
+docker_cmd docker run -d \
  --name "$GW_NAME" \
  --network "$NET_NAME" \
  -e "OPENCLAW_GATEWAY_TOKEN=$TOKEN" \
@@ -115,10 +121,10 @@ EOF
 echo "Waiting for gateway HTTP surface..."
 gateway_ready=0
 for _ in $(seq 1 240); do
-  if [ "$(docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" != "true" ]; then
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" != "true" ]; then
    break
  fi
-  if docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
+  if docker_cmd docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
    const res = await fetch(\"http://127.0.0.1:$PORT/v1/models\", {
      headers: { authorization: \"Bearer $TOKEN\" },
    }).catch(() => null);
@@ -132,16 +138,16 @@ done

 if [ "$gateway_ready" -ne 1 ]; then
  echo "Gateway failed to start"
-  docker inspect "$GW_NAME" --format '{{json .State}}' 2>/dev/null || true
-  if [ "$(docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
-    docker exec "$GW_NAME" bash -lc 'tail -n 200 /tmp/openwebui-gateway.log' || true
+  docker_cmd docker inspect "$GW_NAME" --format '{{json .State}}' 2>/dev/null || true
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
+    docker_cmd docker exec "$GW_NAME" bash -lc 'tail -n 200 /tmp/openwebui-gateway.log' || true
  fi
-  docker logs "$GW_NAME" 2>&1 | tail -n 200 || true
+  docker_cmd docker logs "$GW_NAME" 2>&1 | tail -n 200 || true
  exit 1
 fi

 echo "Starting Open WebUI container..."
-docker run -d \
+docker_cmd docker run -d \
  --name "$OW_NAME" \
  --network "$NET_NAME" \
  -e ENV=prod \
@@ -167,10 +173,10 @@ docker run -d \
 echo "Waiting for Open WebUI..."
 ow_ready=0
 for _ in $(seq 1 240); do
-  if [ "$(docker inspect -f '{{.State.Running}}' "$OW_NAME" 2>/dev/null || echo false)" != "true" ]; then
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$OW_NAME" 2>/dev/null || echo false)" != "true" ]; then
    break
  fi
-  if docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
+  if docker_cmd docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
    const res = await fetch(\"http://$OW_NAME:$WEBUI_PORT/\").catch(() => null);
    process.exit(res && res.status < 500 ? 0 : 1);
  ' >/dev/null 2>&1"; then
@@ -182,17 +188,17 @@ done

 if [ "$ow_ready" -ne 1 ]; then
  echo "Open WebUI failed to start"
-  docker logs "$OW_NAME" 2>&1 | tail -n 200 || true
+  docker_cmd docker logs "$OW_NAME" 2>&1 | tail -n 200 || true
  exit 1
 fi

 echo "Waiting for gateway model endpoint after Open WebUI startup..."
 gateway_model_ready=0
 for _ in $(seq 1 90); do
-  if [ "$(docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" != "true" ]; then
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" != "true" ]; then
    break
  fi
-  if docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
+  if docker_cmd docker exec "$GW_NAME" bash -lc "node --input-type=module -e '
    const controller = new AbortController();
    const timeout = setTimeout(() => controller.abort(), 8000);
    try {
@@ -215,17 +221,17 @@ done

 if [ "$gateway_model_ready" -ne 1 ]; then
  echo "Gateway model endpoint did not stay reachable after Open WebUI startup"
-  docker inspect "$GW_NAME" --format '{{json .State}}' 2>/dev/null || true
-  if [ "$(docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
-    docker exec "$GW_NAME" bash -lc 'tail -n 200 /tmp/openwebui-gateway.log' || true
+  docker_cmd docker inspect "$GW_NAME" --format '{{json .State}}' 2>/dev/null || true
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
+    docker_cmd docker exec "$GW_NAME" bash -lc 'tail -n 200 /tmp/openwebui-gateway.log' || true
  fi
-  docker logs "$GW_NAME" 2>&1 | tail -n 200 || true
-  docker logs "$OW_NAME" 2>&1 | tail -n 200 || true
+  docker_cmd docker logs "$GW_NAME" 2>&1 | tail -n 200 || true
+  docker_cmd docker logs "$OW_NAME" 2>&1 | tail -n 200 || true
  exit 1
 fi

 echo "Running Open WebUI -> OpenClaw smoke..."
-if ! docker exec \
+if ! docker_cmd docker exec \
  -e "OPENWEBUI_BASE_URL=http://$OW_NAME:$WEBUI_PORT" \
  -e "OPENWEBUI_ADMIN_EMAIL=$ADMIN_EMAIL" \
  -e "OPENWEBUI_ADMIN_PASSWORD=$ADMIN_PASSWORD" \
@@ -237,13 +243,13 @@ if ! docker exec \
  node /app/scripts/e2e/openwebui-probe.mjs >/tmp/openwebui-probe.log 2>&1; then
  cat /tmp/openwebui-probe.log 2>/dev/null || true
  echo "Open WebUI probe failed; gateway log tail:"
-  docker inspect "$GW_NAME" --format '{{json .State}}' 2>/dev/null || true
-  if [ "$(docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
-    docker exec "$GW_NAME" bash -lc 'tail -n 200 /tmp/openwebui-gateway.log' || true
+  docker_cmd docker inspect "$GW_NAME" --format '{{json .State}}' 2>/dev/null || true
+  if [ "$(docker_cmd docker inspect -f '{{.State.Running}}' "$GW_NAME" 2>/dev/null || echo false)" = "true" ]; then
+    docker_cmd docker exec "$GW_NAME" bash -lc 'tail -n 200 /tmp/openwebui-gateway.log' || true
  fi
-  docker logs "$GW_NAME" 2>&1 | tail -n 200 || true
+  docker_cmd docker logs "$GW_NAME" 2>&1 | tail -n 200 || true
  echo "Open WebUI container logs:"
-  docker logs "$OW_NAME" 2>&1 | tail -n 200 || true
+  docker_cmd docker logs "$OW_NAME" 2>&1 | tail -n 200 || true
  exit 1
 fi

--- a/scripts/test-docker-all.mjs
+++ b/scripts/test-docker-all.mjs
@@ -277,6 +277,7 @@ function runShellCommand({ command, env, label, logFile, timeoutMs }) {
  return new Promise((resolve) => {
    const child = spawn("bash", ["-lc", command], {
      cwd: ROOT_DIR,
+      detached: process.platform !== "win32",
      env,
      stdio: logFile ? ["ignore", "pipe", "pipe"] : "inherit",
    });
@@ -290,8 +291,8 @@ function runShellCommand({ command, env, label, logFile, timeoutMs }) {
            if (stream) {
              stream.write(`\n==> [${label}] timeout after ${timeoutMs}ms; sending SIGTERM\n`);
            }
-            child.kill("SIGTERM");
-            killTimer = setTimeout(() => child.kill("SIGKILL"), 10_000);
+            terminateChild(child, "SIGTERM");
+            killTimer = setTimeout(() => terminateChild(child, "SIGKILL"), 10_000);
            killTimer.unref?.();
          }, timeoutMs)
        : undefined;
@@ -582,9 +583,21 @@ async function printFailureSummary(failures, tailLines) {
 }

 const activeChildren = new Set();
+function terminateChild(child, signal) {
+  if (process.platform !== "win32" && child.pid) {
+    try {
+      process.kill(-child.pid, signal);
+      return;
+    } catch {
+      // Fall back to killing the direct child below.
+    }
+  }
+  child.kill(signal);
+}
+
 function terminateActiveChildren(signal) {
  for (const child of activeChildren) {
-    child.kill(signal);
+    terminateChild(child, signal);
  }
 }