diff --git a/.github/labeler.yml b/.github/labeler.yml index f3090e13337d..2f612c30056f 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -225,6 +225,14 @@ - "src/daemon/**" - "docs/gateway/**" +"fleet": + - changed-files: + - any-glob-to-any-file: + - "src/fleet/**" + - "src/cli/fleet-cli/**" + - "docs/gateway/multi-tenant-hosting.md" + - "docs/cli/fleet.md" + "docs": - changed-files: - any-glob-to-any-file: diff --git a/docs/cli/fleet.md b/docs/cli/fleet.md new file mode 100644 index 000000000000..917a38a8c01a --- /dev/null +++ b/docs/cli/fleet.md @@ -0,0 +1,250 @@ +--- +summary: "CLI reference for provisioning and managing isolated per-tenant OpenClaw cells" +read_when: + - You host multiple tenant trust domains on one machine + - You need to create, inspect, upgrade, or remove fleet cells +title: "Fleet" +--- + +# `openclaw fleet` + +`openclaw fleet` manages complete OpenClaw instances called **cells**. Each cell has its own Gateway, state, credentials, channel accounts, container, and loopback-only host port. Use one cell for each tenant trust boundary; do not use one shared Gateway as a hostile multi-tenant boundary. + +Fleet is **experimental**. Command names, flags, output shapes, and the container profile can change between releases without a deprecation window while the surface settles. + +Fleet supports Docker and Podman. The default image is `ghcr.io/openclaw/openclaw:latest`. + +## Quick start + +```bash +openclaw fleet create acme +openclaw fleet status acme +openclaw fleet list +``` + +`fleet create` prints the generated Gateway token once along with the cell URL. Store the token immediately, then configure each tenant's channel accounts inside that tenant's cell. + +## Tenant IDs + +Tenant IDs must match: + +```text +^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$ +``` + +This allows 1 to 40 lowercase letters, digits, and internal hyphens. An ID must start and end with a letter or digit. Uppercase letters, underscores, slashes, dots, whitespace, and traversal strings such as `../acme` are rejected. + +The ID becomes part of the container name: `openclaw-cell-`. + +## `fleet create` + +Create a cell and start it: + +```bash +openclaw fleet create acme +``` + +Create a Podman cell on a fixed port without starting it: + +```bash +openclaw fleet create acme \ + --runtime podman \ + --port 19125 \ + --no-start +``` + +Pass tenant-specific environment variables by repeating `--env`: + +```bash +openclaw fleet create acme \ + --env TZ=America/Los_Angeles \ + --env OPENCLAW_DISABLE_BONJOUR=1 +``` + +Environment keys use letters, digits, and underscores and cannot start with a digit. Values must be single-line because Fleet passes them through a protected runtime environment file. Fleet rejects attempts to override the managed container-path and Gateway-token variables listed under [Storage and container layout](#storage-and-container-layout). + +### Create options + +| Option | Default | Description | +| ------------------------- | ------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `--image ` | `ghcr.io/openclaw/openclaw:latest` | Container image for the cell. | +| `--runtime ` | `docker` | Container CLI: `docker` or `podman`. | +| `--port ` | Automatically allocated from `19100` | Loopback host port. An explicitly selected port must not belong to another registered cell. | +| `--memory ` | `2g` | Container memory limit in Docker/Podman syntax. | +| `--cpus ` | `2` | Container CPU limit. | +| `--pids-limit ` | `512` | Maximum number of processes in the container. | +| `--env ` | None | Pass an environment variable to the cell. Repeat for multiple values. | +| `--gateway-token ` | Random 32-character hexadecimal token | Use a supplied Gateway token instead of generating one. See [Token handling](#token-handling). | +| `--no-start` | Cell starts | Create the container without starting it. | +| `--json` | Human-readable output | Print machine-readable output. | + +Automatic allocation selects the first unused registry port at or above `19100`. Fleet rejects duplicate tenant IDs and explicit ports already assigned to another cell. + +Image references are passed as one container-runtime argument. Empty references and values beginning with `-` are rejected so an image cannot be interpreted as a Docker or Podman option. + +The selected Docker or Podman endpoint must be local. Fleet rejects remote Docker contexts, `DOCKER_HOST` endpoints, and remote Podman services before reserving a port or creating local state; remote cell hosts need a separate storage and endpoint contract and are deferred from this MVP. + +The create result includes the tenant ID, container name, host port, Gateway token, and local URL. Even in JSON output, treat the result as secret-bearing because it contains the token. + +## `fleet list` + +List cells in tenant-ID order: + +```bash +openclaw fleet list +openclaw fleet ls +openclaw fleet list --json +``` + +The table contains: + +| Column | Meaning | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tenant` | Tenant ID. | +| `state` | Live container state from Docker or Podman inspection. `unknown` means the runtime was unavailable, or a container with the cell's name exists but its Fleet ownership labels do not match the registry record (a collision or tampering signal — inspect it manually before acting). | +| `port` | Loopback host port mapped to the cell Gateway. | +| `image` | Recorded container image. | +| `created` | Cell creation time. | + +Registry rows remain visible when Docker or Podman is unavailable; only live state becomes `unknown`. + +## `fleet status` + +Inspect one cell: + +```bash +openclaw fleet status acme +openclaw fleet status acme --json +``` + +Status combines the fleet registry row, live container inspection, and a short best-effort request to: + +```text +http://127.0.0.1:/healthz +``` + +The health result is `ok`, `failed`, or `skipped`. `/healthz` proves Gateway liveness, not full readiness of every configured channel or plugin. The probe is skipped when there is no usable local endpoint to check. + +## `fleet start`, `fleet stop`, and `fleet restart` + +Control an existing cell with its recorded runtime: + +```bash +openclaw fleet start acme +openclaw fleet stop acme +openclaw fleet restart acme +``` + +These commands operate on the registered container name. They fail if the tenant is unknown or the recorded runtime cannot perform the operation. + +## `fleet upgrade` + +Re-pull the recorded image and replace the cell container: + +```bash +openclaw fleet upgrade acme +``` + +Move the cell to another image: + +```bash +openclaw fleet upgrade acme --image ghcr.io/openclaw/openclaw: +``` + +Upgrade pulls the target image, inspects the existing container and per-cell network, stops and removes the container, then recreates and starts it. The replacement preserves the same host port, data directories, per-cell bridge network, runtime profile, resource limits, restart policy, Fleet-managed environment, and values originally supplied with `--env`. Mounted state survives container replacement; image-default environment can change with the target image. + +The replacement is committed only after its Gateway answers `/healthz` on the cell's loopback port, matching the health contract the official compose file uses. A replacement that exits, crash-loops, or fails to become healthy within about a minute is removed and the previous container is restored, so a broken image does not take down a working cell. + +The Gateway token is intentionally not stored in the fleet registry. Before removing the old container, Fleet reads its environment and carries `OPENCLAW_GATEWAY_TOKEN` into the replacement. Do not manually remove the old container before an upgrade if the token exists nowhere else you control. + +## `fleet rm` + +Remove a stopped cell from the runtime and registry while keeping tenant data: + +```bash +openclaw fleet rm acme +``` + +A running container requires `--force`: + +```bash +openclaw fleet rm acme --force +``` + +Permanently remove the cell data as well: + +```bash +openclaw fleet rm acme --purge-data --force +``` + +Fleet removes the cell container before removing its dedicated bridge network. `--purge-data` requires `--force`. Before recursive deletion, Fleet resolves both Fleet-owned roots and both per-tenant directories. Each target must be the exact expected tenant leaf, strictly inside its root, and not a symlink. These containment checks prevent a corrupted registry path or cross-tenant symlink from redirecting deletion elsewhere. + +Purge is retryable when an exact expected tenant directory is already absent. This lets a later invocation finish cleanup after a partial filesystem failure without relaxing the path checks for directories that still exist. + +## Storage and container layout + +Cell state and auth-profile encryption keys use separate per-tenant host paths under the active OpenClaw state directory: + +```text +/fleet/cells// +/fleet/auth-profile-secrets// +``` + +The first directory is mounted at `/home/node/.openclaw`. The second is mounted at `/home/node/.config/openclaw`, matching the official Docker setup's encryption-key mount. The encryption key is therefore not exposed beneath the ordinary state mount or included when only the cell-state directory is backed up or shared. Both directories survive normal removal and upgrade; `fleet rm --purge-data --force` deletes both after separate containment checks. + +Before first start, Fleet initializes the cell config with `gateway.mode=local`, token auth, the LAN container bind, and Control UI origins for the allocated host port. The token value is not written to that config; it remains in the container environment. + +Fleet pins the official image's container paths with these environment values: + +| Variable | Container value | +| ------------------------ | ------------------------------------ | +| `HOME` | `/home/node` | +| `OPENCLAW_HOME` | `/home/node` | +| `OPENCLAW_STATE_DIR` | `/home/node/.openclaw` | +| `OPENCLAW_CONFIG_PATH` | `/home/node/.openclaw/openclaw.json` | +| `OPENCLAW_WORKSPACE_DIR` | `/home/node/.openclaw/workspace` | +| `OPENCLAW_GATEWAY_TOKEN` | Generated or supplied cell token | + +The official image defaults to the non-root `node` user with UID 1000. Fleet keeps the private `0700` bind mounts writable without making them world-accessible. Rootful Docker runs the cell with the invoking non-root UID and GID; rootless Docker uses container UID 0, which maps to the invoking unprivileged host user inside the daemon's user namespace. Podman uses `keep-id` with the invoking UID and GID. When Fleet itself runs as root against a rootful runtime, it retains the image user and assigns the initial mount files to UID/GID 1000. + +On SELinux hosts, Docker and Podman mounts receive a private `:Z` relabel. If you restore or relocate cell data, keep the bind-mounted paths writable by the effective container user. The profile is rootless-friendly, but Docker or Podman must already be configured for rootless operation on the host; Fleet does not convert a rootful daemon into a rootless one. + +## Security profile + +Fleet applies the following profile to every cell: + +| Control | Applied profile | Why | +| -------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------------------- | +| Linux capabilities | `--cap-drop=ALL` | The Gateway is a Node.js process and needs no added Linux capabilities. | +| Privilege escalation | `--security-opt no-new-privileges` | Prevents processes from gaining privileges through setuid or setgid binaries. | +| Init process | `--init` | Reaps descendant processes and forwards container lifecycle signals. | +| Process limit | `--pids-limit 512` by default | Bounds fork and process exhaustion. | +| Memory limit | `--memory 2g` by default | Bounds cell memory use. | +| CPU limit | `--cpus 2` by default | Bounds cell CPU use. | +| Restart policy | `--restart unless-stopped` | Restarts a failed cell without overriding an intentional stop. | +| Host publishing | `127.0.0.1::18789` only | Keeps the Gateway off wildcard host interfaces. | +| Cell network | One user-defined bridge per cell | Prevents direct container-IP traffic between cells while retaining outbound NAT access. | +| Container identity | Host-matched user mapping | Keeps private bind mounts writable without granting world access. | +| Persistent state | Per-cell mounts; no shared state mount | Keeps tenant config, credentials, sessions, and workspaces in that tenant's data tree. | +| Container command | `node dist/index.js gateway --bind lan --port 18789` | Listens on the container network so the loopback-only host port mapping can reach it. | + +Fleet never mounts `/var/run/docker.sock`, uses `--privileged` or host networking, or adds capabilities. The per-cell bridge is a cross-cell separation boundary, not an outbound firewall: cells retain the network egress needed for providers and channels. Front the loopback port with a proxy, SSH tunnel, or tailnet configuration that matches your deployment. `http://127.0.0.1:` is directly reachable only from the Fleet host. + +This profile separates tenant containers, but it does not protect tenants from the Fleet operator, the container runtime administrator, or a compromised host. See [Multi-tenant hosting](/gateway/multi-tenant-hosting) for the complete trust model and stronger isolation options. + +## Token handling + +By default, `fleet create` generates a cryptographically random 32-character hexadecimal Gateway token and prints it once in the create result. Store it in your approved secret manager and avoid capturing create output in logs. + +`--gateway-token` places a custom token in the local process arguments, which may be retained in shell history or visible in process listings. Prefer the generated token unless an existing secret-management workflow requires a supplied value. + +The token and every value passed with `--env` live in the container environment. Fleet writes them to a short-lived mode-`0600` environment file, passes only that file's path to Docker or Podman, and removes it after the runtime command finishes. Values explicitly typed in `openclaw fleet create --gateway-token ...` or `--env KEY=VALUE` can still be visible in the outer `openclaw` process arguments and shell history. + +Container environment values are not hidden from the trusted host operator: Docker or Podman administrators can read them with container inspection. Fleet's "shown once" note describes normal CLI output, not resistance to a host administrator. + +## Related + +- [Multi-tenant hosting](/gateway/multi-tenant-hosting) +- [Docker](/install/docker) +- [Podman](/install/podman) +- [Gateway security](/gateway/security) diff --git a/docs/docs.json b/docs/docs.json index 36eb139357f5..4f74f0041278 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1632,7 +1632,8 @@ "gateway/gateway-lock", "gateway/background-process", "gateway/restart-recovery", - "gateway/multiple-gateways" + "gateway/multiple-gateways", + "gateway/multi-tenant-hosting" ] }, { @@ -1744,6 +1745,7 @@ "cli/crestodian", "cli/daemon", "cli/doctor", + "cli/fleet", "cli/gateway", "cli/health", "cli/logs", diff --git a/docs/docs_map.md b/docs/docs_map.md index ae8642916435..3588a72e5bce 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -1499,6 +1499,25 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: macOS: launchctl env overrides - H2: Related +## cli/fleet.md + +- Route: /cli/fleet +- Headings: + - H1: openclaw fleet + - H2: Quick start + - H2: Tenant IDs + - H2: fleet create + - H3: Create options + - H2: fleet list + - H2: fleet status + - H2: fleet start, fleet stop, and fleet restart + - H2: fleet upgrade + - H2: fleet rm + - H2: Storage and container layout + - H2: Security profile + - H2: Token handling + - H2: Related + ## cli/flows.md - Route: /cli/flows @@ -3463,6 +3482,19 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Console formatting (subsystem logging) - H2: Related +## gateway/multi-tenant-hosting.md + +- Route: /gateway/multi-tenant-hosting +- Headings: + - H1: Multi-tenant hosting + - H2: Why each tenant needs a cell + - H2: Architecture + - H2: Trust boundary + - H2: Isolation ladder + - H2: Quick start + - H2: Deferred from the MVP + - H2: Related + ## gateway/multiple-gateways.md - Route: /gateway/multiple-gateways diff --git a/docs/gateway/multi-tenant-hosting.md b/docs/gateway/multi-tenant-hosting.md new file mode 100644 index 000000000000..e6872e2365e3 --- /dev/null +++ b/docs/gateway/multi-tenant-hosting.md @@ -0,0 +1,104 @@ +--- +summary: "Host multiple tenant trust domains as one isolated OpenClaw Gateway cell per tenant" +read_when: + - You are hosting OpenClaw for multiple users or organizations + - You need to choose an isolation boundary for tenant workloads +title: "Multi-tenant hosting" +--- + +# Multi-tenant hosting + +OpenClaw's default security model is one trusted operator boundary per Gateway, not hostile multi-tenant isolation inside one shared Gateway. Hosting users or organizations that do not share a trust boundary therefore means running a separate complete OpenClaw instance for each tenant. + +`openclaw fleet` calls each isolated instance a **cell**. A cell is a full Gateway in a hardened container with its own state, credentials, workspace, channel accounts, token, and loopback-only host port. + +Fleet is **experimental**: its commands, flags, and container profile can change between releases without a deprecation window while the surface settles. + +## Why each tenant needs a cell + +An authenticated operator inside one Gateway has a trusted control-plane role. Session IDs select routing; they do not authorize one tenant against another. Agent sandboxing can reduce the effect of untrusted content and tool execution, but it does not turn one shared Gateway into a tenant authorization boundary. + +Use one cell per tenant so each trust domain has a separate Gateway process, container, persistent state tree, and Gateway credential. This follows the [Gateway security model](/gateway/security): do not co-locate mutually untrusted users in one OpenClaw process or one OS user. + +## Architecture + +The Fleet CLI is a host-side lifecycle supervisor. It records cells in the OpenClaw state database and asks a local Docker or Podman runtime to create, inspect, start, stop, replace, and remove their containers. Remote runtime endpoints are rejected because Fleet's bind paths and loopback URLs belong to the local host; remote cell hosts are deferred until they have an explicit storage and endpoint contract. Fleet does not proxy tenant messages and does not add a shared application-level data path between cells. + +Each cell runs the official `ghcr.io/openclaw/openclaw` image on its own user-defined bridge network. Separate bridges prevent direct container-IP traffic between cells while retaining outbound NAT access for providers and channels. This is not an egress firewall; apply host or runtime network policy when a tenant needs outbound restrictions. The cell Gateway listens on port `18789` inside the container, while the runtime publishes it only to `127.0.0.1:` on the host. An operator can place an approved reverse proxy, SSH tunnel, or tailnet in front of that loopback endpoint when remote access is needed. + +Persistent Gateway state comes from `/fleet/cells//` and is mounted at `/home/node/.openclaw`. Auth-profile encryption keys come from the separate `/fleet/auth-profile-secrets//` host path and are mounted at `/home/node/.config/openclaw`, matching the official [Docker persistence layout](/install/docker#storage-and-persistence). The key is not nested beneath the ordinary state mount. Per-tenant channel accounts terminate inside the cell that owns them, so there is no shared channel account or shared inbound message router in the Fleet MVP. + +The official image defaults to the non-root `node` user with UID 1000. Fleet uses host-compatible user mappings so private bind mounts stay writable: Podman uses `keep-id`, rootful Docker uses the invoking non-root identity, and rootless Docker maps container root to the unprivileged daemon user. Docker and Podman apply a private `:Z` relabel when host SELinux is active. The container profile avoids privileged host features and is rootless-friendly, but rootless operation is a host runtime choice and prerequisite, not something Fleet enables automatically. + +## Trust boundary + +Multi-tenancy protects tenants from each other. The Fleet operator and the host are trusted by every tenant. Resistance to a compromised host is a non-goal. + +This means a host administrator can inspect container configuration and environment, read mounted cell data, replace images, or enter containers. Gateway tokens and values passed with `--env` are visible to an administrator through Docker or Podman inspection. Use host controls, administrative access policy, monitoring, backups, and an approved secret manager accordingly. + +The baseline prevents accidental wildcard network exposure and removes common container escalation primitives, but it does not make an untrusted host safe. + +## Isolation ladder + +Choose the boundary that matches the tenants you host: + +1. **Hardened container baseline.** Fleet drops all Linux capabilities, enables `no-new-privileges`, applies PID, memory, and CPU limits, uses separate persistent mounts and bridge networks, and publishes only to host loopback. This is the MVP profile for tenants that trust the operator and host. +2. **Stronger container or VM isolation.** For higher-risk workloads, configure Docker or Podman to use a stronger OCI isolation runtime such as gVisor or Kata Containers, or place cells in microVMs. This is runtime or infrastructure configuration; Fleet's `--runtime docker|podman` option chooses the container CLI, not the OCI isolation backend. See Docker's [alternative container runtimes](https://docs.docker.com/engine/daemon/alternative-runtimes/) and the [Docker VM runtime guide](/install/docker-vm-runtime). +3. **Separate machines for hostile tenants.** Do not co-locate hostile tenants in one OpenClaw process or OS user. When tenants do not trust the same host operator or need a stronger administrative boundary, use separate VMs or physical hosts with separate runtime administration. + +No rung in this ladder changes the OpenClaw application trust model: one Gateway remains one trusted operator domain. + +## Quick start + +Create a cell. The command prints a generated Gateway token once, so store it immediately: + +```bash +openclaw fleet create acme +``` + +Open the reported `http://127.0.0.1:` URL on the Fleet host, authenticate with that tenant's token, and configure provider credentials and channel accounts inside the cell. + +Check the container state and Gateway liveness: + +```bash +openclaw fleet status acme +``` + +Upgrade while preserving the host port, mounted data, resource profile, user-supplied environment, and Gateway token: + +```bash +openclaw fleet upgrade acme +``` + +Remove the container and registry row while retaining tenant data: + +```bash +openclaw fleet rm acme --force +``` + +To delete persistent tenant data too, add `--purge-data`. Purge requires `--force`, is irreversible, and performs a resolved-path containment check before deleting anything: + +```bash +openclaw fleet rm acme --purge-data --force +``` + +See the [`openclaw fleet` CLI reference](/cli/fleet) for every command and option. + +## Deferred from the MVP + +The first Fleet release deliberately leaves these surfaces to later designs: + +- Shared channel accounts or a shared ingress router +- Slimmed-down per-tenant host processes instead of complete OpenClaw instances +- Remote cell hosts managed by one supervisor +- A tenant self-service portal, billing plane, or delegated administration UI + +These features need explicit identity, routing, authorization, and failure-domain contracts. They should not be approximated by sharing one Gateway or its credentials across tenants. They are also not Fleet's lane: Fleet stays a single-host lifecycle supervisor, and multi-machine, identity-governed fleets belong to a dedicated control-plane layer above it. + +## Related + +- [`openclaw fleet`](/cli/fleet) +- [Gateway security](/gateway/security) +- [Multiple gateways](/gateway/multiple-gateways) +- [Docker](/install/docker) +- [Podman](/install/podman) diff --git a/docs/gateway/security/index.md b/docs/gateway/security/index.md index 29da0b620ac8..4f98a428a25a 100644 --- a/docs/gateway/security/index.md +++ b/docs/gateway/security/index.md @@ -24,6 +24,8 @@ title: "Security" - Inside one Gateway, authenticated operator access is a trusted control-plane role, not a per-user tenant role. - `sessionKey` (session IDs, labels) is a routing selector, not an authorization token. +Hosting multiple users or organizations? Run one isolated Gateway cell per tenant instead of sharing a Gateway. See [Multi-tenant hosting](/gateway/multi-tenant-hosting). + Before changing remote access, DM policy, reverse proxy, or public exposure, run through the [Gateway exposure runbook](/gateway/security/exposure-runbook) as a pre-flight/rollback checklist. ## `openclaw security audit` diff --git a/docs/install/docker.md b/docs/install/docker.md index 06dc18cac6c4..01de13593158 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -10,6 +10,8 @@ Docker is **optional**. Use it for an isolated, throwaway gateway environment or The default sandbox backend uses Docker when `agents.defaults.sandbox` is enabled, but sandboxing is off by default and does not require the gateway itself to run in Docker. SSH and OpenShell sandbox backends are also available; see [Sandboxing](/gateway/sandboxing). +Hosting multiple users? See [Multi-tenant hosting](/gateway/multi-tenant-hosting) for the one-cell-per-tenant model. + ## Prerequisites - Docker Desktop (or Docker Engine) + Docker Compose v2 diff --git a/src/cli/command-catalog.ts b/src/cli/command-catalog.ts index 264ea5e464c8..75ad190bd30b 100644 --- a/src/cli/command-catalog.ts +++ b/src/cli/command-catalog.ts @@ -276,6 +276,10 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ commandPath: ["worktrees"], policy: { loadPlugins: "never", networkProxy: "bypass" }, }, + { + commandPath: ["fleet"], + policy: { loadPlugins: "never", networkProxy: "bypass" }, + }, { commandPath: ["doctor"], policy: { diff --git a/src/cli/fleet-cli.ts b/src/cli/fleet-cli.ts new file mode 100644 index 000000000000..7f40ee2c4941 --- /dev/null +++ b/src/cli/fleet-cli.ts @@ -0,0 +1 @@ +export { registerFleetCli } from "./fleet-cli/register.js"; diff --git a/src/cli/fleet-cli/commands.runtime.test.ts b/src/cli/fleet-cli/commands.runtime.test.ts new file mode 100644 index 000000000000..8d3ce654d2c0 --- /dev/null +++ b/src/cli/fleet-cli/commands.runtime.test.ts @@ -0,0 +1,125 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = await vi.hoisted(async () => { + const { createCliRuntimeMock } = await import("../test-runtime-mock.js"); + return { + ...createCliRuntimeMock(vi), + create: vi.fn(), + list: vi.fn(), + status: vi.fn(), + lifecycle: vi.fn(), + upgrade: vi.fn(), + remove: vi.fn(), + }; +}); + +vi.mock("../../runtime.js", () => ({ defaultRuntime: mocks.defaultRuntime })); +vi.mock("../../fleet/service.runtime.js", () => ({ + createFleetService: () => ({ + create: mocks.create, + list: mocks.list, + status: mocks.status, + lifecycle: mocks.lifecycle, + upgrade: mocks.upgrade, + remove: mocks.remove, + }), +})); + +import { + runFleetCreateCommand, + runFleetListCommand, + runFleetRemoveCommand, + runFleetStatusCommand, +} from "./commands.runtime.js"; + +describe("fleet command output", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.runtimeLogs.length = 0; + mocks.runtimeErrors.length = 0; + }); + + it("writes the documented secret-bearing create JSON shape", async () => { + const result = { + tenant: "acme", + containerName: "openclaw-cell-acme", + port: 19_100, + image: "ghcr.io/openclaw/openclaw:latest", + runtime: "docker" as const, + started: true, + token: "gw-token", + tokenNote: "Shown once. Store this Gateway token securely.", + url: "http://127.0.0.1:19100", + nextStep: + "Open http://127.0.0.1:19100, then configure per-tenant channel accounts inside the cell.", + }; + mocks.create.mockResolvedValue(result); + + await runFleetCreateCommand({ tenant: "acme", json: true }); + + expect(mocks.defaultRuntime.writeJson).toHaveBeenCalledWith(result); + }); + + it("prints the Gateway token exactly once in human create output", async () => { + mocks.create.mockResolvedValue({ + tenant: "acme", + containerName: "openclaw-cell-acme", + port: 19_100, + image: "image", + runtime: "docker", + started: true, + token: "one-token", + tokenNote: "Shown once. Store this Gateway token securely.", + url: "http://127.0.0.1:19100", + nextStep: "Open the cell.", + }); + + await runFleetCreateCommand({ tenant: "acme", json: false }); + + expect(mocks.runtimeLogs.join("\n").match(/one-token/gu)).toHaveLength(1); + expect(mocks.runtimeLogs.join("\n")).toContain("Shown once"); + }); + + it("wraps deterministic list JSON in a cells object", async () => { + const cells = [ + { + tenant: "acme", + state: "running", + port: 19_100, + image: "image", + created: "2026-01-01T00:00:00.000Z", + }, + ]; + mocks.list.mockResolvedValue(cells); + + await runFleetListCommand({ json: true }); + + expect(mocks.defaultRuntime.writeJson).toHaveBeenCalledWith({ cells }); + }); + + it("writes status JSON and describes retained data on removal", async () => { + const status = { + tenant: "acme", + containerName: "openclaw-cell-acme", + runtime: "docker" as const, + port: 19_100, + image: "image", + created: "2026-01-01T00:00:00.000Z", + dataDir: "/tmp/acme", + container: { state: "running", running: true, managed: true }, + health: { + status: "ok" as const, + url: "http://127.0.0.1:19100/healthz", + httpStatus: 200, + }, + }; + mocks.status.mockResolvedValue(status); + mocks.remove.mockResolvedValue({ tenant: "acme", action: "rm", dataPurged: false }); + + await runFleetStatusCommand({ tenant: "acme", json: true }); + await runFleetRemoveCommand({ tenant: "acme", force: false, purgeData: false }); + + expect(mocks.defaultRuntime.writeJson).toHaveBeenCalledWith(status); + expect(mocks.runtimeLogs).toContain("Removed fleet cell acme; data retained."); + }); +}); diff --git a/src/cli/fleet-cli/commands.runtime.ts b/src/cli/fleet-cli/commands.runtime.ts new file mode 100644 index 000000000000..f92ec3ceec90 --- /dev/null +++ b/src/cli/fleet-cli/commands.runtime.ts @@ -0,0 +1,116 @@ +// Runtime-backed fleet command handlers and human/JSON output formatting. +import { getTerminalTableWidth, renderTable } from "../../../packages/terminal-core/src/table.js"; +import { + createFleetService, + type FleetCreateOptions, + type FleetHealthResult, + type FleetLifecycleAction, +} from "../../fleet/service.runtime.js"; +import { defaultRuntime } from "../../runtime.js"; + +const fleetService = createFleetService(); + +export async function runFleetCreateCommand( + options: FleetCreateOptions & { json: boolean }, +): Promise { + const result = await fleetService.create(options); + if (options.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(`Tenant: ${result.tenant}`); + defaultRuntime.log(`Container: ${result.containerName}`); + defaultRuntime.log(`Port: ${result.port}`); + defaultRuntime.log(`Token: ${result.token}`); + defaultRuntime.log(result.tokenNote); + defaultRuntime.log(`Next: ${result.nextStep}`); +} + +export async function runFleetListCommand(options: { json: boolean }): Promise { + const cells = await fleetService.list(); + if (options.json) { + defaultRuntime.writeJson({ cells }); + return; + } + if (cells.length === 0) { + defaultRuntime.log("No fleet cells."); + return; + } + defaultRuntime.log( + renderTable({ + width: getTerminalTableWidth(), + columns: [ + { key: "tenant", header: "Tenant", minWidth: 10, flex: true }, + { key: "state", header: "State", minWidth: 10 }, + { key: "port", header: "Port", minWidth: 7 }, + { key: "image", header: "Image", minWidth: 24, flex: true }, + { key: "created", header: "Created", minWidth: 24 }, + ], + rows: cells.map((cell) => ({ + tenant: cell.tenant, + state: cell.state, + port: String(cell.port), + image: cell.image, + created: cell.created, + })), + }).trimEnd(), + ); +} + +function formatHealth(health: FleetHealthResult): string { + if (health.status === "ok") { + return `ok (HTTP ${health.httpStatus})`; + } + if (health.status === "failed") { + return `failed (${health.error})`; + } + return `skipped (${health.reason})`; +} + +export async function runFleetStatusCommand(options: { + tenant: string; + json: boolean; +}): Promise { + const result = await fleetService.status(options.tenant); + if (options.json) { + defaultRuntime.writeJson(result); + return; + } + defaultRuntime.log(`Tenant: ${result.tenant}`); + defaultRuntime.log(`Container: ${result.containerName}`); + defaultRuntime.log(`State: ${result.container.state}`); + defaultRuntime.log(`Port: ${result.port}`); + defaultRuntime.log(`Image: ${result.image}`); + defaultRuntime.log(`Created: ${result.created}`); + defaultRuntime.log(`Data: ${result.dataDir}`); + defaultRuntime.log(`Health: ${formatHealth(result.health)}`); +} + +export async function runFleetLifecycleCommand(options: { + tenant: string; + action: FleetLifecycleAction; +}): Promise { + const result = await fleetService.lifecycle(options.tenant, options.action); + defaultRuntime.log(`${result.action} complete for fleet cell ${result.tenant}.`); +} + +export async function runFleetUpgradeCommand(options: { + tenant: string; + image?: string; +}): Promise { + const result = await fleetService.upgrade(options.tenant, options.image); + defaultRuntime.log(`Upgraded fleet cell ${result.tenant} to ${result.image}.`); +} + +export async function runFleetRemoveCommand(options: { + tenant: string; + purgeData: boolean; + force: boolean; +}): Promise { + const result = await fleetService.remove(options); + defaultRuntime.log( + result.dataPurged + ? `Removed fleet cell ${result.tenant} and purged its data.` + : `Removed fleet cell ${result.tenant}; data retained.`, + ); +} diff --git a/src/cli/fleet-cli/register.test.ts b/src/cli/fleet-cli/register.test.ts new file mode 100644 index 000000000000..c134c00cba13 --- /dev/null +++ b/src/cli/fleet-cli/register.test.ts @@ -0,0 +1,148 @@ +import { Command } from "commander"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { registerFleetCli } from "../fleet-cli.js"; + +const mocks = await vi.hoisted(async () => { + const { createCliRuntimeMock } = await import("../test-runtime-mock.js"); + return { + ...createCliRuntimeMock(vi), + runFleetCreateCommand: vi.fn(), + runFleetListCommand: vi.fn(), + runFleetStatusCommand: vi.fn(), + runFleetLifecycleCommand: vi.fn(), + runFleetUpgradeCommand: vi.fn(), + runFleetRemoveCommand: vi.fn(), + }; +}); + +vi.mock("../../runtime.js", () => ({ + defaultRuntime: mocks.defaultRuntime, +})); + +vi.mock("./commands.runtime.js", () => ({ + runFleetCreateCommand: mocks.runFleetCreateCommand, + runFleetListCommand: mocks.runFleetListCommand, + runFleetStatusCommand: mocks.runFleetStatusCommand, + runFleetLifecycleCommand: mocks.runFleetLifecycleCommand, + runFleetUpgradeCommand: mocks.runFleetUpgradeCommand, + runFleetRemoveCommand: mocks.runFleetRemoveCommand, +})); + +function createProgram(): Command { + const program = new Command(); + program.exitOverride(); + program.configureOutput({ + writeErr: () => undefined, + writeOut: () => undefined, + }); + registerFleetCli(program); + return program; +} + +async function runFleetCli(argv: string[]): Promise { + await createProgram().parseAsync(["fleet", ...argv], { from: "user" }); +} + +describe("fleet cli", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.runtimeLogs.length = 0; + mocks.runtimeErrors.length = 0; + }); + + it("normalizes create options", async () => { + await runFleetCli([ + "create", + "tenant-a", + "--image", + "registry.example/openclaw:v1", + "--runtime", + "podman", + "--port", + "19123", + "--memory", + "3g", + "--cpus", + "1.5", + "--pids-limit", + "256", + "--env", + "CHANNEL_ID=alpha", + "--env", + "FEATURE_FLAG=1", + "--gateway-token", + "test-token", + "--no-start", + "--json", + ]); + + expect(mocks.runFleetCreateCommand).toHaveBeenCalledOnce(); + expect(mocks.runFleetCreateCommand).toHaveBeenCalledWith({ + tenant: "tenant-a", + image: "registry.example/openclaw:v1", + runtime: "podman", + port: 19_123, + memory: "3g", + cpus: "1.5", + pidsLimit: 256, + env: ["CHANNEL_ID=alpha", "FEATURE_FLAG=1"], + gatewayToken: "test-token", + start: false, + json: true, + }); + }); + + it("applies create defaults", async () => { + await runFleetCli(["create", "tenant-b"]); + + expect(mocks.runFleetCreateCommand).toHaveBeenCalledWith({ + tenant: "tenant-b", + image: "ghcr.io/openclaw/openclaw:latest", + runtime: "docker", + port: undefined, + memory: "2g", + cpus: "2", + pidsLimit: 512, + env: [], + gatewayToken: undefined, + start: true, + json: false, + }); + }); + + it.each(["list", "ls"])("routes fleet %s JSON output", async (command) => { + await runFleetCli([command, "--json"]); + + expect(mocks.runFleetListCommand).toHaveBeenCalledOnce(); + expect(mocks.runFleetListCommand).toHaveBeenCalledWith({ json: true }); + }); + + it("normalizes remove safety flags", async () => { + await runFleetCli(["rm", "tenant-a", "--purge-data", "--force"]); + + expect(mocks.runFleetRemoveCommand).toHaveBeenCalledOnce(); + expect(mocks.runFleetRemoveCommand).toHaveBeenCalledWith({ + tenant: "tenant-a", + purgeData: true, + force: true, + }); + }); + + it.each([ + { + argv: ["create", "tenant-a", "--runtime", "containerd"], + error: /--runtime must be docker or podman/, + }, + { + argv: ["create", "tenant-a", "--port", "65536"], + error: /--port must be between 1 and 65535/, + }, + { + argv: ["create", "tenant-a", "--cpus", "0"], + error: /--cpus must be a positive number/, + }, + ])("rejects invalid create options: $argv", async ({ argv, error }) => { + await expect(runFleetCli(argv)).rejects.toThrow(error); + expect(mocks.runFleetCreateCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/src/cli/fleet-cli/register.ts b/src/cli/fleet-cli/register.ts new file mode 100644 index 000000000000..2820be8b5ea4 --- /dev/null +++ b/src/cli/fleet-cli/register.ts @@ -0,0 +1,139 @@ +import { InvalidArgumentError, type Command } from "commander"; +import { createLazyImportLoader } from "../../shared/lazy-promise.js"; +import { collectOption, parseStrictPositiveIntOption } from "../program/helpers.js"; + +type FleetRuntimeModule = typeof import("./commands.runtime.js"); + +const fleetRuntimeLoader = createLazyImportLoader( + () => import("./commands.runtime.js"), +); + +function loadFleetRuntime(): Promise { + return fleetRuntimeLoader.load(); +} + +function parseContainerRuntime(value: string): "docker" | "podman" { + if (value === "docker" || value === "podman") { + return value; + } + throw new InvalidArgumentError("--runtime must be docker or podman."); +} + +function parsePort(value: string): number { + const port = parseStrictPositiveIntOption(value, "--port"); + if (port > 65_535) { + throw new InvalidArgumentError("--port must be between 1 and 65535."); + } + return port; +} + +function parseCpus(value: string): string { + const cpus = Number(value); + if (!Number.isFinite(cpus) || cpus <= 0) { + throw new InvalidArgumentError("--cpus must be a positive number."); + } + return value; +} + +export function registerFleetCli(program: Command): void { + const fleet = program + .command("fleet") + .description("Provision and manage isolated tenant cells (experimental)"); + + fleet + .command("create") + .description("Create an isolated tenant cell") + .argument("", "Tenant slug") + .option("--image ", "Container image", "ghcr.io/openclaw/openclaw:latest") + .option( + "--runtime ", + "Container runtime (docker or podman)", + parseContainerRuntime, + "docker", + ) + .option("--port ", "Host loopback port (default: allocate from 19100)", parsePort) + .option("--memory ", "Container memory limit", "2g") + .option("--cpus ", "Container CPU limit", parseCpus, "2") + .option( + "--pids-limit ", + "Container process limit", + (value: string) => parseStrictPositiveIntOption(value, "--pids-limit"), + 512, + ) + .option("--env ", "Pass an environment variable to the cell", collectOption, []) + .option("--gateway-token ", "Use an existing Gateway token") + .option("--no-start", "Create the container without starting it") + .option("--json", "Output JSON", false) + .action( + async ( + tenant: string, + options: { + image: string; + runtime: "docker" | "podman"; + port?: number; + memory: string; + cpus: string; + pidsLimit: number; + env: string[]; + gatewayToken?: string; + start: boolean; + json: boolean; + }, + ) => { + const runtime = await loadFleetRuntime(); + await runtime.runFleetCreateCommand({ tenant, ...options }); + }, + ); + + fleet + .command("list") + .alias("ls") + .description("List tenant cells") + .option("--json", "Output JSON", false) + .action(async (options: { json: boolean }) => { + const runtime = await loadFleetRuntime(); + await runtime.runFleetListCommand(options); + }); + + fleet + .command("status") + .description("Show tenant cell status") + .argument("", "Tenant slug") + .option("--json", "Output JSON", false) + .action(async (tenant: string, options: { json: boolean }) => { + const runtime = await loadFleetRuntime(); + await runtime.runFleetStatusCommand({ tenant, ...options }); + }); + + for (const action of ["start", "stop", "restart"] as const) { + fleet + .command(action) + .description(`${action[0]?.toUpperCase()}${action.slice(1)} a tenant cell`) + .argument("", "Tenant slug") + .action(async (tenant: string) => { + const runtime = await loadFleetRuntime(); + await runtime.runFleetLifecycleCommand({ action, tenant }); + }); + } + + fleet + .command("upgrade") + .description("Replace a tenant cell with a freshly pulled image") + .argument("", "Tenant slug") + .option("--image ", "Replacement image (default: recorded image)") + .action(async (tenant: string, options: { image?: string }) => { + const runtime = await loadFleetRuntime(); + await runtime.runFleetUpgradeCommand({ tenant, ...options }); + }); + + fleet + .command("rm") + .description("Remove a tenant cell") + .argument("", "Tenant slug") + .option("--purge-data", "Delete the tenant data directory", false) + .option("--force", "Remove a running cell", false) + .action(async (tenant: string, options: { purgeData: boolean; force: boolean }) => { + const runtime = await loadFleetRuntime(); + await runtime.runFleetRemoveCommand({ tenant, ...options }); + }); +} diff --git a/src/cli/program/register.subclis-core.ts b/src/cli/program/register.subclis-core.ts index 2dfc0728b177..5f8a1121220d 100644 --- a/src/cli/program/register.subclis-core.ts +++ b/src/cli/program/register.subclis-core.ts @@ -167,6 +167,11 @@ const entrySpecs: readonly CommandGroupDescriptorSpec[] = [ loadModule: () => import("../sandbox-cli.js"), exportName: "registerSandboxCli", }, + { + commandNames: ["fleet"], + loadModule: () => import("../fleet-cli.js"), + exportName: "registerFleetCli", + }, { commandNames: ["worktrees"], loadModule: () => import("../worktrees-cli.js"), diff --git a/src/cli/program/subcli-descriptors.ts b/src/cli/program/subcli-descriptors.ts index 3bb6a4d8ed75..591072febc9d 100644 --- a/src/cli/program/subcli-descriptors.ts +++ b/src/cli/program/subcli-descriptors.ts @@ -81,6 +81,11 @@ const subCliCommandCatalog = defineCommandDescriptorCatalog([ description: "Manage sandbox containers (Docker-based agent isolation)", hasSubcommands: true, }, + { + name: "fleet", + description: "Provision and manage isolated tenant cells (experimental)", + hasSubcommands: true, + }, { name: "worktrees", description: "Create, inspect, restore, and clean up managed worktrees", diff --git a/src/fleet/cell-profile.test.ts b/src/fleet/cell-profile.test.ts new file mode 100644 index 000000000000..ee6652ac58ec --- /dev/null +++ b/src/fleet/cell-profile.test.ts @@ -0,0 +1,275 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + allocateHostPort, + buildCellCreateArgs, + buildCellEnvironment, + buildCellRunArgs, + cellAuthSecretDir, + cellContainerName, + cellDataDir, + cellNetworkName, + cellOwnerId, + DEFAULT_FLEET_IMAGE, + FLEET_BASE_PORT, + FLEET_ATTEMPT_LABEL, + FLEET_CONTAINER_AUTH_SECRET_DIR, + FLEET_CONTAINER_STATE_DIR, + FLEET_ENV_KEYS_LABEL, + FLEET_GATEWAY_PORT, + FLEET_OWNER_LABEL, + FLEET_TENANT_LABEL, + parseEnvAssignments, + type CellContainerProfile, + validateFleetImage, + validateTenantId, +} from "./cell-profile.js"; + +const TEST_ENVIRONMENT_FILE = "/tmp/openclaw-fleet-env/cell.env"; + +function makeProfile(overrides: Partial = {}): CellContainerProfile { + const tenantId = overrides.tenantId ?? "acme"; + const dataDir = overrides.dataDir ?? "/tmp/openclaw/fleet/cells/acme"; + return { + tenantId, + containerName: overrides.containerName ?? cellContainerName(tenantId), + networkName: overrides.networkName ?? cellNetworkName(tenantId), + image: DEFAULT_FLEET_IMAGE, + runtime: "docker", + hostPort: FLEET_BASE_PORT, + dataDir, + authSecretDir: overrides.authSecretDir ?? "/tmp/openclaw/fleet/auth-profile-secrets/acme", + ownerId: overrides.ownerId ?? cellOwnerId(dataDir), + attemptId: overrides.attemptId ?? "11111111111111111111111111111111", + memory: "2g", + cpus: "2", + pidsLimit: 512, + environment: buildCellEnvironment("gateway-token", { TENANT_REGION: "west=1" }), + selinuxRelabel: false, + ...overrides, + }; +} + +function expectOption(args: string[], flag: string, value: string): void { + const index = args.indexOf(flag); + expect(index, `${flag} should be present`).toBeGreaterThanOrEqual(0); + expect(args[index + 1]).toBe(value); +} + +describe("fleet tenant ids", () => { + it.each(["a", "0", "acme", "acme-2", "a--b", "a".repeat(40)])("accepts %s", (tenantId) => { + expect(validateTenantId(tenantId)).toBe(tenantId); + }); + + it.each([ + "", + ".", + "..", + "../acme", + "acme/other", + "acme\\other", + "-acme", + "acme-", + "Acme", + "acme_team", + "acme team", + "ténant", + "a".repeat(41), + ])("rejects %j", (tenantId) => { + expect(() => validateTenantId(tenantId)).toThrow(/tenant id/i); + }); + + it("derives stable container and data paths", () => { + expect(cellContainerName("acme-2")).toBe("openclaw-cell-acme-2"); + expect(cellNetworkName("acme-2")).toBe("openclaw-cell-acme-2-net"); + expect(cellDataDir("/srv/openclaw", "acme-2")).toBe( + path.join("/srv/openclaw", "fleet", "cells", "acme-2"), + ); + expect(cellAuthSecretDir("/srv/openclaw", "acme-2")).toBe( + path.join("/srv/openclaw", "fleet", "auth-profile-secrets", "acme-2"), + ); + }); +}); + +describe("fleet port allocation", () => { + it("starts at the fleet base port and scans gaps", () => { + expect(allocateHostPort([])).toBe(FLEET_BASE_PORT); + expect(allocateHostPort([FLEET_BASE_PORT, FLEET_BASE_PORT + 2])).toBe(FLEET_BASE_PORT + 1); + }); + + it("honors an explicit free TCP port", () => { + expect(allocateHostPort([FLEET_BASE_PORT], 8080)).toBe(8080); + }); + + it("rejects explicit collisions and invalid ports", () => { + expect(() => allocateHostPort([20_000], 20_000)).toThrow(/already allocated/i); + for (const port of [0, -1, 65_536, 1.5, Number.NaN]) { + expect(() => allocateHostPort([], port)).toThrow(/1 to 65535/i); + } + }); +}); + +describe("fleet image references", () => { + it("normalizes valid references and rejects option-like values", () => { + expect(validateFleetImage(" ghcr.io/openclaw/openclaw:v1 ")).toBe( + "ghcr.io/openclaw/openclaw:v1", + ); + expect(() => validateFleetImage("--help")).toThrow(/must not begin/iu); + expect(() => validateFleetImage(" ")).toThrow(/must not be empty/iu); + }); +}); + +describe("fleet cell environment", () => { + it("parses KEY=VAL assignments without truncating values", () => { + expect(parseEnvAssignments(["REGION=west", "URL=https://example.test/?a=b", "EMPTY="])).toEqual( + { + REGION: "west", + URL: "https://example.test/?a=b", + EMPTY: "", + }, + ); + expect(parseEnvAssignments(["REGION=west", "REGION=east"])).toEqual({ REGION: "east" }); + }); + + it.each(["MISSING", "=value", "BAD-KEY=value", "1BAD=value", "HAS SPACE=value"])( + "rejects malformed assignment %j", + (assignment) => { + expect(() => parseEnvAssignments([assignment])).toThrow(/--env/i); + }, + ); + + it.each([ + "HOME", + "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", + "OPENCLAW_CONFIG_PATH", + "OPENCLAW_WORKSPACE_DIR", + "OPENCLAW_GATEWAY_TOKEN", + ])("rejects reserved variable %s", (key) => { + expect(() => parseEnvAssignments([`${key}=override`])).toThrow(/fleet-managed/i); + }); + + it("pins the documented container paths and token", () => { + expect(buildCellEnvironment("secret", { REGION: "west" })).toEqual({ + HOME: "/home/node", + OPENCLAW_HOME: "/home/node", + OPENCLAW_STATE_DIR: FLEET_CONTAINER_STATE_DIR, + OPENCLAW_CONFIG_PATH: `${FLEET_CONTAINER_STATE_DIR}/openclaw.json`, + OPENCLAW_WORKSPACE_DIR: `${FLEET_CONTAINER_STATE_DIR}/workspace`, + OPENCLAW_GATEWAY_TOKEN: "secret", + REGION: "west", + }); + }); + + it("rejects values that cannot be represented in the protected environment file", () => { + expect(() => buildCellEnvironment("bad\ntoken", {})).toThrow(/line breaks/iu); + expect(() => buildCellEnvironment("token", { FEATURE: "line\nbreak" })).toThrow( + /must be one line/iu, + ); + }); +}); + +describe("fleet container arguments", () => { + it("builds the complete hardened run profile", () => { + const profile = makeProfile(); + const args = buildCellRunArgs(profile, { environmentFile: TEST_ENVIRONMENT_FILE }); + + expect(args.slice(0, 2)).toEqual(["run", "-d"]); + expectOption(args, "--name", "openclaw-cell-acme"); + expectOption(args, "--label", `${FLEET_TENANT_LABEL}=acme`); + expect(args).toContain(`${FLEET_OWNER_LABEL}=${cellOwnerId(profile.dataDir)}`); + expect(args).toContain(`${FLEET_ATTEMPT_LABEL}=${profile.attemptId}`); + expect(args).toContain(`${FLEET_ENV_KEYS_LABEL}=TENANT_REGION`); + expect(args).toContain("--init"); + expect(args).toContain("--cap-drop=ALL"); + expectOption(args, "--security-opt", "no-new-privileges"); + expectOption(args, "--pids-limit", "512"); + expectOption(args, "--memory", "2g"); + expectOption(args, "--cpus", "2"); + expectOption(args, "--restart", "unless-stopped"); + expectOption(args, "--network", "openclaw-cell-acme-net"); + expectOption(args, "-p", `127.0.0.1:${FLEET_BASE_PORT}:${FLEET_GATEWAY_PORT}`); + expect(args).toContain(`${profile.dataDir}:${FLEET_CONTAINER_STATE_DIR}`); + expect(args).toContain(`${profile.authSecretDir}:${FLEET_CONTAINER_AUTH_SECRET_DIR}`); + expectOption(args, "--env-file", TEST_ENVIRONMENT_FILE); + expect(args.join(" ")).not.toContain("gateway-token"); + expect(args.join(" ")).not.toContain("west=1"); + expect(args.slice(-8)).toEqual([ + DEFAULT_FLEET_IMAGE, + "node", + "dist/index.js", + "gateway", + "--bind", + "lan", + "--port", + String(FLEET_GATEWAY_PORT), + ]); + }); + + it("builds a stopped container with the same profile", () => { + const args = buildCellCreateArgs(makeProfile(), { + environmentFile: TEST_ENVIRONMENT_FILE, + }); + expect(args[0]).toBe("create"); + expect(args).not.toContain("-d"); + expect(args).toContain("--cap-drop=ALL"); + expect(args.slice(-7)).toEqual([ + "node", + "dist/index.js", + "gateway", + "--bind", + "lan", + "--port", + String(FLEET_GATEWAY_PORT), + ]); + }); + + it("adds rootless keep-id arguments and private SELinux labels for Podman", () => { + const args = buildCellRunArgs( + makeProfile({ + runtime: "podman", + containerUser: { mode: "podman-keep-id", uid: 501, gid: 20 }, + selinuxRelabel: true, + }), + { environmentFile: TEST_ENVIRONMENT_FILE }, + ); + expect(args).toContain("--userns=keep-id"); + expectOption(args, "--user", "501:20"); + expect(args).toContain(`${makeProfile().dataDir}:${FLEET_CONTAINER_STATE_DIR}:Z`); + expect(args).toContain(`${makeProfile().authSecretDir}:${FLEET_CONTAINER_AUTH_SECRET_DIR}:Z`); + }); + + it("adds private SELinux labels for Docker mounts", () => { + const profile = makeProfile({ selinuxRelabel: true }); + const args = buildCellRunArgs(profile, { environmentFile: TEST_ENVIRONMENT_FILE }); + + expect(args).toContain(`${profile.dataDir}:${FLEET_CONTAINER_STATE_DIR}:Z`); + expect(args).toContain(`${profile.authSecretDir}:${FLEET_CONTAINER_AUTH_SECRET_DIR}:Z`); + }); + + it("runs rootful Docker with the invoking non-root identity", () => { + const args = buildCellRunArgs( + makeProfile({ containerUser: { mode: "numeric", uid: 1001, gid: 1002 } }), + { environmentFile: TEST_ENVIRONMENT_FILE }, + ); + + expectOption(args, "--user", "1001:1002"); + expect(args).not.toContain("--userns=keep-id"); + }); + + it("never enables privileged host integration", () => { + for (const args of [ + buildCellRunArgs(makeProfile(), { environmentFile: TEST_ENVIRONMENT_FILE }), + buildCellCreateArgs(makeProfile(), { environmentFile: TEST_ENVIRONMENT_FILE }), + ]) { + const rendered = args.join(" "); + expect(rendered).not.toContain("docker.sock"); + expect(args).not.toContain("--privileged"); + expect(args).not.toContain("--network=host"); + expectOption(args, "--network", "openclaw-cell-acme-net"); + expect(args[args.indexOf("--network") + 1]).not.toBe("host"); + expect(rendered).not.toContain("--cap-add"); + expect(rendered).not.toContain("0.0.0.0"); + } + }); +}); diff --git a/src/fleet/cell-profile.ts b/src/fleet/cell-profile.ts new file mode 100644 index 000000000000..35e58bf37b84 --- /dev/null +++ b/src/fleet/cell-profile.ts @@ -0,0 +1,310 @@ +import crypto from "node:crypto"; +import path from "node:path"; + +export type FleetContainerRuntimeName = "docker" | "podman"; + +export const DEFAULT_FLEET_IMAGE = "ghcr.io/openclaw/openclaw:latest"; +export const FLEET_BASE_PORT = 19_100; +export const FLEET_GATEWAY_PORT = 18_789; +export const FLEET_CONTAINER_HOME = "/home/node"; +export const FLEET_CONTAINER_STATE_DIR = "/home/node/.openclaw"; +export const FLEET_CONTAINER_AUTH_SECRET_DIR = "/home/node/.config/openclaw"; +export const FLEET_TENANT_LABEL = "openclaw.fleet.tenant"; +export const FLEET_OWNER_LABEL = "openclaw.fleet.owner"; +export const FLEET_ATTEMPT_LABEL = "openclaw.fleet.attempt"; +export const FLEET_ENV_KEYS_LABEL = "openclaw.fleet.env-keys"; +export const FLEET_MANAGED_ENV_KEYS = [ + "HOME", + "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", + "OPENCLAW_CONFIG_PATH", + "OPENCLAW_WORKSPACE_DIR", + "OPENCLAW_GATEWAY_TOKEN", +] as const; + +const FLEET_TENANT_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$/; +const ENV_KEY_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const MAX_TCP_PORT = 65_535; +const RESERVED_ENV_KEYS = new Set(FLEET_MANAGED_ENV_KEYS); + +function hasInvalidEnvironmentFileValue(value: string): boolean { + return value.includes("\n") || value.includes("\r") || value.includes("\u0000"); +} + +export interface CellContainerProfile { + tenantId: string; + containerName: string; + networkName: string; + image: string; + runtime: FleetContainerRuntimeName; + hostPort: number; + dataDir: string; + authSecretDir: string; + ownerId: string; + attemptId: string; + memory: string; + cpus: string; + pidsLimit: number; + environment: Readonly>; + containerUser?: + | { mode: "numeric"; uid: number; gid: number } + | { mode: "podman-keep-id"; uid: number; gid: number }; + selinuxRelabel: boolean; +} + +export type CellContainerArgOptions = { + environmentFile: string; +}; + +export function validateTenantId(tenantId: string): string { + if (!FLEET_TENANT_PATTERN.test(tenantId)) { + throw new Error( + "Invalid tenant id. Use 1-40 lowercase letters, digits, or hyphens; start and end with a letter or digit.", + ); + } + return tenantId; +} + +export function validateFleetImage(image: string): string { + const normalized = image.trim(); + if (!normalized) { + throw new Error("Fleet container image must not be empty."); + } + if (normalized.startsWith("-")) { + throw new Error("Fleet container image must not begin with '-'."); + } + return normalized; +} + +function validateHostPort(port: number): number { + if (!Number.isInteger(port) || port < 1 || port > MAX_TCP_PORT) { + throw new Error("Host port must be an integer from 1 to 65535."); + } + return port; +} + +export function allocateHostPort(usedPorts: Iterable, requestedPort?: number): number { + const used = new Set(usedPorts); + if (requestedPort !== undefined) { + const port = validateHostPort(requestedPort); + if (used.has(port)) { + throw new Error(`Host port ${port} is already allocated to another fleet cell.`); + } + return port; + } + + for (let port = FLEET_BASE_PORT; port <= MAX_TCP_PORT; port += 1) { + if (!used.has(port)) { + return port; + } + } + throw new Error(`No free fleet host ports remain from ${FLEET_BASE_PORT} to ${MAX_TCP_PORT}.`); +} + +export function parseEnvAssignments(values: string[]): Record { + const environment: Record = {}; + for (const assignment of values) { + const separator = assignment.indexOf("="); + if (separator <= 0) { + throw new Error("Invalid --env value; expected KEY=VAL."); + } + const key = assignment.slice(0, separator); + if (!ENV_KEY_PATTERN.test(key)) { + throw new Error("Invalid --env key; use letters, digits, and underscores."); + } + if (RESERVED_ENV_KEYS.has(key)) { + throw new Error(`--env cannot override fleet-managed variable ${key}.`); + } + environment[key] = assignment.slice(separator + 1); + } + return environment; +} + +export function buildCellEnvironment( + token: string, + userEnv: Readonly>, +): Record { + if (hasInvalidEnvironmentFileValue(token)) { + throw new Error("Gateway token must not contain line breaks or null bytes."); + } + for (const key of Object.keys(userEnv)) { + if (!ENV_KEY_PATTERN.test(key)) { + throw new Error(`Invalid cell environment key: ${key}`); + } + if (RESERVED_ENV_KEYS.has(key)) { + throw new Error(`Cell environment cannot override fleet-managed variable ${key}.`); + } + if (hasInvalidEnvironmentFileValue(userEnv[key] ?? "")) { + throw new Error(`Cell environment value for ${key} must be one line.`); + } + } + return { + HOME: FLEET_CONTAINER_HOME, + OPENCLAW_HOME: FLEET_CONTAINER_HOME, + OPENCLAW_STATE_DIR: FLEET_CONTAINER_STATE_DIR, + OPENCLAW_CONFIG_PATH: `${FLEET_CONTAINER_STATE_DIR}/openclaw.json`, + OPENCLAW_WORKSPACE_DIR: `${FLEET_CONTAINER_STATE_DIR}/workspace`, + OPENCLAW_GATEWAY_TOKEN: token, + ...userEnv, + }; +} + +export function cellContainerName(tenantId: string): string { + return `openclaw-cell-${validateTenantId(tenantId)}`; +} + +export function cellNetworkName(tenantId: string): string { + return `${cellContainerName(tenantId)}-net`; +} + +export function cellDataDir(stateDir: string, tenantId: string): string { + return path.join(stateDir, "fleet", "cells", validateTenantId(tenantId)); +} + +export function cellAuthSecretDir(stateDir: string, tenantId: string): string { + return path.join(stateDir, "fleet", "auth-profile-secrets", validateTenantId(tenantId)); +} + +export function cellOwnerId(dataDir: string): string { + return crypto.createHash("sha256").update(path.resolve(dataDir)).digest("hex").slice(0, 32); +} + +export function validateCellContainerProfile(profile: CellContainerProfile): void { + validateTenantId(profile.tenantId); + validateHostPort(profile.hostPort); + if (profile.containerName !== cellContainerName(profile.tenantId)) { + throw new Error(`Fleet container name must be ${cellContainerName(profile.tenantId)}.`); + } + if (profile.networkName !== cellNetworkName(profile.tenantId)) { + throw new Error(`Fleet network name must be ${cellNetworkName(profile.tenantId)}.`); + } + if (validateFleetImage(profile.image) !== profile.image) { + throw new Error("Fleet container image must not have surrounding whitespace."); + } + if (!profile.dataDir.trim()) { + throw new Error("Fleet cell data directory must not be empty."); + } + if (!profile.authSecretDir.trim()) { + throw new Error("Fleet cell auth-secret directory must not be empty."); + } + if (profile.ownerId !== cellOwnerId(profile.dataDir)) { + throw new Error("Fleet cell owner id does not match its data directory."); + } + if (!/^[a-f0-9]{32}$/u.test(profile.attemptId)) { + throw new Error("Fleet cell attempt id must be 32 lowercase hexadecimal characters."); + } + if (!profile.memory.trim()) { + throw new Error("Fleet cell memory limit must not be empty."); + } + if (!profile.cpus.trim()) { + throw new Error("Fleet cell CPU limit must not be empty."); + } + const cpus = Number(profile.cpus); + if (!Number.isFinite(cpus) || cpus <= 0) { + throw new Error("Fleet cell CPU limit must be a positive number."); + } + if (!Number.isInteger(profile.pidsLimit) || profile.pidsLimit < 1) { + throw new Error("Fleet cell PID limit must be a positive integer."); + } + if (profile.containerUser) { + if ( + !Number.isInteger(profile.containerUser.uid) || + profile.containerUser.uid < 0 || + !Number.isInteger(profile.containerUser.gid) || + profile.containerUser.gid < 0 + ) { + throw new Error("Container uid and gid must be non-negative integers."); + } + if (profile.containerUser.mode === "podman-keep-id" && profile.runtime !== "podman") { + throw new Error("keep-id user mapping requires Podman."); + } + } + for (const [key, value] of Object.entries(profile.environment)) { + if (!ENV_KEY_PATTERN.test(key) || hasInvalidEnvironmentFileValue(value)) { + throw new Error(`Invalid fleet cell environment entry: ${key}`); + } + } +} + +function buildCellContainerArgs( + profile: CellContainerProfile, + operation: "run" | "create", + options: CellContainerArgOptions, +): string[] { + validateCellContainerProfile(profile); + if (!options.environmentFile.trim()) { + throw new Error("Fleet cell environment file path must not be empty."); + } + const containerUserArgs = profile.containerUser + ? [ + ...(profile.containerUser.mode === "podman-keep-id" ? ["--userns=keep-id"] : []), + "--user", + `${profile.containerUser.uid}:${profile.containerUser.gid}`, + ] + : []; + const userEnvironmentKeys = Object.keys(profile.environment) + .filter((key) => !RESERVED_ENV_KEYS.has(key)) + .toSorted(); + const mountSuffix = profile.selinuxRelabel ? ":Z" : ""; + + return [ + operation, + ...(operation === "run" ? ["-d"] : []), + "--name", + profile.containerName, + "--label", + `${FLEET_TENANT_LABEL}=${profile.tenantId}`, + "--label", + `${FLEET_OWNER_LABEL}=${profile.ownerId}`, + "--label", + `${FLEET_ATTEMPT_LABEL}=${profile.attemptId}`, + "--label", + `${FLEET_ENV_KEYS_LABEL}=${userEnvironmentKeys.join(",")}`, + "--init", + ...containerUserArgs, + // The official image runs a plain Node process, so the cell needs no Linux capabilities. + "--cap-drop=ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + String(profile.pidsLimit), + "--memory", + profile.memory, + "--cpus", + profile.cpus, + "--restart", + "unless-stopped", + "--network", + profile.networkName, + "-p", + `127.0.0.1:${profile.hostPort}:${FLEET_GATEWAY_PORT}`, + "--volume", + `${profile.dataDir}:${FLEET_CONTAINER_STATE_DIR}${mountSuffix}`, + "--volume", + `${profile.authSecretDir}:${FLEET_CONTAINER_AUTH_SECRET_DIR}${mountSuffix}`, + "--env-file", + options.environmentFile, + profile.image, + "node", + "dist/index.js", + "gateway", + "--bind", + "lan", + "--port", + String(FLEET_GATEWAY_PORT), + ]; +} + +export function buildCellRunArgs( + profile: CellContainerProfile, + options: CellContainerArgOptions, +): string[] { + return buildCellContainerArgs(profile, "run", options); +} + +export function buildCellCreateArgs( + profile: CellContainerProfile, + options: CellContainerArgOptions, +): string[] { + return buildCellContainerArgs(profile, "create", options); +} diff --git a/src/fleet/containers.runtime.test.ts b/src/fleet/containers.runtime.test.ts new file mode 100644 index 000000000000..e88cfd8cec9f --- /dev/null +++ b/src/fleet/containers.runtime.test.ts @@ -0,0 +1,422 @@ +import fs from "node:fs/promises"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const profileMocks = vi.hoisted(() => ({ + buildCellRunArgs: vi.fn((_profile: unknown, options: { environmentFile: string }) => [ + "run", + "--env-file", + options.environmentFile, + "cell-image", + ]), + buildCellCreateArgs: vi.fn((_profile: unknown, options: { environmentFile: string }) => [ + "create", + "--env-file", + options.environmentFile, + "cell-image", + ]), + validateFleetImage: vi.fn((image: string) => image), +})); + +vi.mock("./cell-profile.js", () => profileMocks); + +import { + createFleetContainerRuntime, + type CellContainerProfile, + type FleetContainerCommandExecutor, +} from "./containers.runtime.js"; + +function successfulExecutor() { + return vi.fn(async () => ({ + stdout: "", + stderr: "", + code: 0, + })); +} + +describe("fleet container runtime", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("accepts only local Docker endpoints", async () => { + const localExecutor = vi.fn(async () => ({ + stdout: JSON.stringify([ + { Endpoints: { docker: { Host: "unix:///run/user/1000/docker.sock" } } }, + ]), + stderr: "", + code: 0, + })); + await expect( + createFleetContainerRuntime(localExecutor).assertLocal("docker"), + ).resolves.toBeUndefined(); + expect(localExecutor).toHaveBeenCalledWith("docker", ["context", "inspect"], {}); + + const remoteExecutor = vi.fn(async () => ({ + stdout: JSON.stringify([ + { Endpoints: { docker: { Host: "tcp://remote.example.invalid:2376" } } }, + ]), + stderr: "", + code: 0, + })); + + await expect(createFleetContainerRuntime(remoteExecutor).assertLocal("docker")).rejects.toThrow( + /local Docker endpoint.*remote cells/iu, + ); + expect(remoteExecutor).toHaveBeenCalledWith("docker", ["context", "inspect"], {}); + }); + + it.each([ + [false, true], + [true, false], + ] as const)("classifies Podman serviceIsRemote=%s", async (serviceIsRemote, accepted) => { + const executor = vi.fn(async () => ({ + stdout: JSON.stringify({ host: { serviceIsRemote } }), + stderr: "", + code: 0, + })); + const check = createFleetContainerRuntime(executor).assertLocal("podman"); + + if (accepted) { + await expect(check).resolves.toBeUndefined(); + } else { + await expect(check).rejects.toThrow(/local Podman.*remote cells/iu); + } + expect(executor).toHaveBeenCalledWith("podman", ["info", "--format", "json"], {}); + }); + + it("fails closed on malformed runtime-locality output", async () => { + const executor = vi.fn(async () => ({ + stdout: "{}", + stderr: "", + code: 0, + })); + + await expect(createFleetContainerRuntime(executor).assertLocal("docker")).rejects.toThrow( + /invalid response/iu, + ); + await expect(createFleetContainerRuntime(executor).assertLocal("podman")).rejects.toThrow( + /invalid response/iu, + ); + }); + + it("uses run or create args according to the requested start mode", async () => { + const environmentFiles: string[] = []; + const executor = vi.fn(async (_runtime, args) => { + const environmentFile = args[args.indexOf("--env-file") + 1]; + if (!environmentFile) { + throw new Error("missing environment file"); + } + environmentFiles.push(environmentFile); + await expect(fs.readFile(environmentFile, "utf8")).resolves.toBe( + "OPENCLAW_GATEWAY_TOKEN=fake-value\n", + ); + return { stdout: "", stderr: "", code: 0 }; + }); + const runtime = createFleetContainerRuntime(executor); + const profile = { + runtime: "podman", + environment: { OPENCLAW_GATEWAY_TOKEN: "fake-value" }, + } as unknown as CellContainerProfile; + + await runtime.run(profile, true); + await runtime.run(profile, false); + + expect(profileMocks.buildCellRunArgs).toHaveBeenCalledWith(profile, { + environmentFile: environmentFiles[0], + }); + expect(profileMocks.buildCellCreateArgs).toHaveBeenCalledWith(profile, { + environmentFile: environmentFiles[1], + }); + expect(executor).toHaveBeenNthCalledWith( + 1, + "podman", + ["run", "--env-file", environmentFiles[0], "cell-image"], + { redactValues: ["fake-value"] }, + ); + expect(executor).toHaveBeenNthCalledWith( + 2, + "podman", + ["create", "--env-file", environmentFiles[1], "cell-image"], + { + redactValues: ["fake-value"], + }, + ); + await Promise.all( + environmentFiles.map(async (environmentFile) => { + await expect(fs.stat(environmentFile)).rejects.toMatchObject({ code: "ENOENT" }); + }), + ); + }); + + it("normalizes Docker and Podman inspect output", async () => { + const executor = vi.fn(async () => ({ + stdout: JSON.stringify([ + { + Image: "sha256:old-image-id", + State: { Status: "running", Running: true }, + Config: { + Env: ["OPENCLAW_GATEWAY_TOKEN=secret", "FEATURE=a=b"], + Image: "ghcr.io/openclaw/openclaw:latest", + Labels: { "openclaw.fleet.tenant": "acme" }, + User: "1000:1000", + }, + HostConfig: { + Memory: 2_147_483_648, + NanoCpus: 2_000_000_000, + PidsLimit: 512, + UsernsMode: "keep-id", + }, + }, + ]), + stderr: "", + code: 0, + })); + + await expect( + createFleetContainerRuntime(executor).inspect("podman", "cell-acme"), + ).resolves.toEqual({ + kind: "ok", + state: "running", + running: true, + labels: { "openclaw.fleet.tenant": "acme" }, + environment: { OPENCLAW_GATEWAY_TOKEN: "secret", FEATURE: "a=b" }, + imageId: "sha256:old-image-id", + memory: "2147483648", + cpus: "2", + pidsLimit: 512, + user: "1000:1000", + usernsMode: "keep-id", + }); + expect(executor).toHaveBeenCalledWith("podman", ["container", "inspect", "cell-acme"], { + allowFailure: true, + }); + }); + + it("detects a rootless Docker daemon from security options", async () => { + const executor = vi.fn(async () => ({ + stdout: JSON.stringify(["name=seccomp,profile=builtin", "name=rootless"]), + stderr: "", + code: 0, + })); + + await expect(createFleetContainerRuntime(executor).isDockerRootless()).resolves.toBe(true); + expect(executor).toHaveBeenCalledWith( + "docker", + ["info", "--format", "{{json .SecurityOptions}}"], + {}, + ); + }); + + it("distinguishes missing containers from unavailable runtimes", async () => { + const missingExecutor = vi.fn(async () => ({ + stdout: "", + stderr: "Error: no such container: cell-missing", + code: 1, + })); + const unavailableExecutor = vi.fn(async () => ({ + stdout: "", + stderr: "Cannot connect to the Docker daemon", + code: 1, + })); + + await expect( + createFleetContainerRuntime(missingExecutor).inspect("docker", "cell-missing"), + ).resolves.toEqual({ kind: "missing", state: "missing" }); + await expect( + createFleetContainerRuntime(unavailableExecutor).inspect("docker", "cell-acme"), + ).resolves.toEqual({ + kind: "unavailable", + state: "unknown", + error: "Cannot connect to the Docker daemon", + }); + }); + + it.each([ + ["docker", "Labels", "Containers", "Name"], + ["podman", "labels", "containers", "name"], + ] as const)( + "normalizes %s network labels and attached containers", + async (runtimeName, labelsField, containersField, nameField) => { + const executor = vi.fn(async () => ({ + stdout: JSON.stringify([ + { + [labelsField]: { + "openclaw.fleet.tenant": "acme", + "openclaw.fleet.owner": "owner-id", + }, + [containersField]: { + "container-b": { [nameField]: "peer-b" }, + "container-a": { [nameField]: "peer-a" }, + }, + }, + ]), + stderr: "", + code: 0, + })); + + await expect( + createFleetContainerRuntime(executor).inspectNetwork(runtimeName, "openclaw-cell-acme-net"), + ).resolves.toEqual({ + kind: "ok", + labels: { + "openclaw.fleet.tenant": "acme", + "openclaw.fleet.owner": "owner-id", + }, + attachedContainers: [ + { id: "container-a", name: "peer-a" }, + { id: "container-b", name: "peer-b" }, + ], + }); + expect(executor).toHaveBeenCalledWith( + runtimeName, + ["network", "inspect", "openclaw-cell-acme-net"], + { allowFailure: true }, + ); + }, + ); + + it("normalizes a network with no attached containers", async () => { + const executor = vi.fn(async () => ({ + stdout: JSON.stringify([{ Labels: {}, Containers: {} }]), + stderr: "", + code: 0, + })); + + await expect( + createFleetContainerRuntime(executor).inspectNetwork("docker", "openclaw-cell-acme-net"), + ).resolves.toEqual({ kind: "ok", labels: {}, attachedContainers: [] }); + }); + + it("distinguishes missing networks from unavailable runtimes", async () => { + const missingExecutor = vi.fn(async () => ({ + stdout: "", + stderr: "Error: network openclaw-cell-missing-net not found", + code: 1, + })); + const unavailableExecutor = vi.fn(async () => ({ + stdout: "", + stderr: "Cannot connect to the Docker daemon", + code: 1, + })); + + await expect( + createFleetContainerRuntime(missingExecutor).inspectNetwork( + "docker", + "openclaw-cell-missing-net", + ), + ).resolves.toEqual({ kind: "missing" }); + await expect( + createFleetContainerRuntime(unavailableExecutor).inspectNetwork( + "docker", + "openclaw-cell-acme-net", + ), + ).resolves.toEqual({ + kind: "unavailable", + error: "Cannot connect to the Docker daemon", + }); + }); + + it("treats malformed network inspect JSON as unavailable", async () => { + const executor = vi.fn(async () => ({ + stdout: JSON.stringify([{ Labels: { "openclaw.fleet.tenant": 42 } }]), + stderr: "", + code: 0, + })); + + await expect( + createFleetContainerRuntime(executor).inspectNetwork("docker", "openclaw-cell-acme-net"), + ).resolves.toEqual({ + kind: "unavailable", + error: "network inspect returned an invalid response", + }); + }); + + it("treats malformed inspect JSON as unavailable without echoing its output", async () => { + const executor = vi.fn(async () => ({ + stdout: 'not-json OPENCLAW_GATEWAY_TOKEN="secret"', + stderr: "", + code: 0, + })); + + const result = await createFleetContainerRuntime(executor).inspect("docker", "cell-acme"); + + expect(result).toEqual({ + kind: "unavailable", + state: "unknown", + error: "container inspect returned an invalid response", + }); + expect(JSON.stringify(result)).not.toContain("secret"); + }); + + it("redacts environment values from command failures", async () => { + const executor = vi.fn(async () => { + throw new Error("runtime rejected fake-value"); + }); + const runtime = createFleetContainerRuntime(executor); + const profile = { + runtime: "docker", + environment: { OPENCLAW_GATEWAY_TOKEN: "fake-value" }, + } as unknown as CellContainerProfile; + + let failure: unknown; + try { + await runtime.run(profile, true); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain(""); + expect((failure as Error).message).not.toContain("fake-value"); + }); + + it("executes lifecycle commands with the selected runtime", async () => { + const executor = successfulExecutor(); + const runtime = createFleetContainerRuntime(executor); + + await runtime.pull("docker", "image:v2"); + await runtime.start("docker", "cell-acme"); + await runtime.stop("docker", "cell-acme"); + await runtime.restart("docker", "cell-acme"); + await runtime.remove("docker", "cell-acme", true); + await runtime.remove("docker", "cell-acme", false); + + expect(executor.mock.calls.map(([, args]) => args)).toEqual([ + ["pull", "image:v2"], + ["start", "cell-acme"], + ["stop", "cell-acme"], + ["restart", "cell-acme"], + ["rm", "--force", "cell-acme"], + ["rm", "cell-acme"], + ]); + }); + + it("creates and removes a labeled per-cell network", async () => { + const executor = successfulExecutor(); + const runtime = createFleetContainerRuntime(executor); + + await runtime.createNetwork("podman", "openclaw-cell-acme-net", { + "openclaw.fleet.tenant": "acme", + "openclaw.fleet.attempt": "attempt-id", + "openclaw.fleet.owner": "owner-id", + }); + await runtime.removeNetwork("podman", "openclaw-cell-acme-net"); + + expect(executor.mock.calls.map(([, args]) => args)).toEqual([ + [ + "network", + "create", + "--driver", + "bridge", + "--label", + "openclaw.fleet.attempt=attempt-id", + "--label", + "openclaw.fleet.owner=owner-id", + "--label", + "openclaw.fleet.tenant=acme", + "openclaw-cell-acme-net", + ], + ["network", "rm", "openclaw-cell-acme-net"], + ]); + }); +}); diff --git a/src/fleet/containers.runtime.ts b/src/fleet/containers.runtime.ts new file mode 100644 index 000000000000..0b549532b2e8 --- /dev/null +++ b/src/fleet/containers.runtime.ts @@ -0,0 +1,591 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { runCommandWithTimeout } from "../process/exec.js"; +import { + buildCellCreateArgs, + buildCellRunArgs, + validateFleetImage, + type CellContainerProfile, + type FleetContainerRuntimeName, +} from "./cell-profile.js"; + +export type { CellContainerProfile, FleetContainerRuntimeName } from "./cell-profile.js"; + +export type FleetContainerCommandOptions = { + allowFailure?: boolean; + redactValues?: readonly string[]; +}; + +export type FleetContainerCommandResult = { + stdout: string; + stderr: string; + code: number; +}; + +export type FleetContainerCommandExecutor = ( + runtime: FleetContainerRuntimeName, + args: string[], + options: FleetContainerCommandOptions, +) => Promise; + +export type FleetContainerInspectResult = + | { + kind: "ok"; + state: string; + running: boolean; + labels: Record; + environment: Record; + imageId: string; + memory: string; + cpus: string; + pidsLimit: number | undefined; + user?: string; + usernsMode?: string; + } + | { kind: "missing"; state: "missing" } + | { kind: "unavailable"; state: "unknown"; error: string }; + +export type FleetNetworkInspectResult = + | { + kind: "ok"; + labels: Record; + attachedContainers: Array<{ id: string; name?: string }>; + } + | { kind: "missing" } + | { kind: "unavailable"; error: string }; + +export type FleetContainerRuntime = ReturnType; + +const COMMAND_TIMEOUT_MS = 10 * 60_000; +const COMMAND_MAX_OUTPUT_BYTES = 4 * 1024 * 1024; + +class InvalidInspectOutputError extends Error { + constructor() { + super("container inspect returned an invalid response"); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireRecord(value: unknown): Record { + if (!isRecord(value)) { + throw new InvalidInspectOutputError(); + } + return value; +} + +function requireString(value: unknown): string { + if (typeof value !== "string" || value.length === 0) { + throw new InvalidInspectOutputError(); + } + return value; +} + +function requireBoolean(value: unknown): boolean { + if (typeof value !== "boolean") { + throw new InvalidInspectOutputError(); + } + return value; +} + +function requireNonNegativeNumber(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new InvalidInspectOutputError(); + } + return value; +} + +function readOptionalString(value: unknown): string | undefined { + if (value === undefined || value === null || value === "") { + return undefined; + } + if (typeof value !== "string") { + throw new InvalidInspectOutputError(); + } + return value; +} + +function readLabels(value: unknown): Record { + if (value === undefined || value === null) { + return {}; + } + const record = requireRecord(value); + const labels: Record = {}; + for (const [key, label] of Object.entries(record)) { + if (typeof label !== "string") { + throw new InvalidInspectOutputError(); + } + labels[key] = label; + } + return labels; +} + +function readNetworkAttachments(value: unknown): Array<{ id: string; name?: string }> { + if (value === undefined || value === null) { + return []; + } + const record = requireRecord(value); + return Object.entries(record) + .map(([id, rawAttachment]) => { + if (!id) { + throw new InvalidInspectOutputError(); + } + const attachment = requireRecord(rawAttachment); + const name = readOptionalString(attachment.Name ?? attachment.name); + const normalized: { id: string; name?: string } = { id }; + if (name) { + normalized.name = name; + } + return normalized; + }) + .toSorted((left, right) => left.id.localeCompare(right.id)); +} + +function readEnvironment(value: unknown): Record { + if (value === undefined || value === null) { + return {}; + } + if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) { + throw new InvalidInspectOutputError(); + } + const environment: Record = {}; + for (const assignment of value) { + const separator = assignment.indexOf("="); + if (separator <= 0) { + throw new InvalidInspectOutputError(); + } + environment[assignment.slice(0, separator)] = assignment.slice(separator + 1); + } + return environment; +} + +function readPidsLimit(value: unknown): number | undefined { + if (value === undefined || value === null || value === 0) { + return undefined; + } + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new InvalidInspectOutputError(); + } + return value; +} + +function parseInspectOutput(stdout: string): Extract { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new InvalidInspectOutputError(); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new InvalidInspectOutputError(); + } + + const inspected = requireRecord(parsed[0]); + const state = requireRecord(inspected.State); + const config = requireRecord(inspected.Config); + const hostConfig = requireRecord(inspected.HostConfig); + const nanoCpus = requireNonNegativeNumber(hostConfig.NanoCpus); + const user = readOptionalString(config.User); + const usernsMode = readOptionalString(hostConfig.UsernsMode); + + return { + kind: "ok", + state: requireString(state.Status), + running: requireBoolean(state.Running), + labels: readLabels(config.Labels), + environment: readEnvironment(config.Env), + imageId: requireString(inspected.Image), + memory: String(requireNonNegativeNumber(hostConfig.Memory)), + cpus: String(nanoCpus / 1_000_000_000), + pidsLimit: readPidsLimit(hostConfig.PidsLimit), + ...(user ? { user } : {}), + ...(usernsMode ? { usernsMode } : {}), + }; +} + +function parseNetworkInspectOutput( + stdout: string, +): Extract { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new InvalidInspectOutputError(); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new InvalidInspectOutputError(); + } + + const inspected = requireRecord(parsed[0]); + return { + kind: "ok", + labels: readLabels(inspected.Labels ?? inspected.labels), + attachedContainers: readNetworkAttachments(inspected.Containers ?? inspected.containers), + }; +} + +function parseDockerContextEndpoint(stdout: string): string { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("docker context inspect returned an invalid response"); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("docker context inspect returned an invalid response"); + } + try { + const context = requireRecord(parsed[0]); + const endpoints = requireRecord(context.Endpoints); + const docker = requireRecord(endpoints.docker); + return requireString(docker.Host); + } catch { + throw new Error("docker context inspect returned an invalid response"); + } +} + +function isLocalDockerEndpoint(endpoint: string): boolean { + const normalized = endpoint.toLowerCase(); + const unixPrefix = "unix:///"; + if (normalized.startsWith(unixPrefix)) { + return normalized.length > unixPrefix.length; + } + const windowsPipePrefix = "npipe:////./pipe/"; + return ( + process.platform === "win32" && + normalized.startsWith(windowsPipePrefix) && + normalized.length > windowsPipePrefix.length + ); +} + +function parsePodmanServiceIsRemote(stdout: string): boolean { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("podman info returned an invalid response"); + } + try { + const info = requireRecord(parsed); + const host = requireRecord(info.host); + const serviceIsRemote = host.serviceIsRemote; + if (typeof serviceIsRemote !== "boolean") { + throw new Error(); + } + return serviceIsRemote; + } catch { + throw new Error("podman info returned an invalid response"); + } +} + +function parseDockerRootlessInfo(stdout: string): boolean { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("docker info returned an invalid security-options response"); + } + if (!Array.isArray(parsed) || !parsed.every((entry) => typeof entry === "string")) { + throw new Error("docker info returned an invalid security-options response"); + } + return parsed.some((entry) => entry.split(",").includes("name=rootless")); +} + +function readEnvironmentValues(args: readonly string[], extraValues?: readonly string[]): string[] { + const values: string[] = []; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + let assignment: string | undefined; + if (arg === "--env" || arg === "-e") { + assignment = args[index + 1]; + index += 1; + } else if (arg.startsWith("--env=")) { + assignment = arg.slice("--env=".length); + } else if (arg.startsWith("-e=")) { + assignment = arg.slice("-e=".length); + } + if (!assignment) { + continue; + } + const separator = assignment.indexOf("="); + if (separator >= 0 && separator < assignment.length - 1) { + values.push(assignment.slice(separator + 1)); + } + } + for (const value of extraValues ?? []) { + if (value) { + values.push(value); + } + } + return [...new Set(values)]; +} + +function redactEnvironmentValues( + text: string, + args: readonly string[], + extraValues?: readonly string[], +): string { + let redacted = text; + const values = readEnvironmentValues(args, extraValues).toSorted( + (left, right) => right.length - left.length, + ); + for (const value of values) { + redacted = redacted.replaceAll(value, ""); + } + return redacted; +} + +function formatExecutorError( + error: unknown, + runtime: FleetContainerRuntimeName, + args: readonly string[], + extraValues?: readonly string[], +): Error { + const detail = + error instanceof Error ? redactEnvironmentValues(error.message, args, extraValues).trim() : ""; + return new Error(detail || `${runtime} container command failed`); +} + +function commandFailureError( + runtime: FleetContainerRuntimeName, + args: readonly string[], + result: FleetContainerCommandResult, + extraValues?: readonly string[], +): Error { + const detail = redactEnvironmentValues(result.stderr, args, extraValues).trim(); + return new Error(detail || `${runtime} container command failed with exit code ${result.code}`); +} + +const defaultFleetContainerCommandExecutor: FleetContainerCommandExecutor = async ( + runtime, + args, + options, +) => { + const result = await runCommandWithTimeout([runtime, ...args], { + timeoutMs: COMMAND_TIMEOUT_MS, + maxOutputBytes: COMMAND_MAX_OUTPUT_BYTES, + }); + const normalized = { + stdout: result.stdout, + stderr: redactEnvironmentValues(result.stderr, args, options.redactValues), + code: result.code ?? 1, + }; + if (normalized.code !== 0 && !options.allowFailure) { + throw commandFailureError(runtime, args, normalized, options.redactValues); + } + return normalized; +}; + +function isMissingContainerError(stderr: string): boolean { + const normalized = stderr.toLowerCase(); + return ( + normalized.includes("no such object") || + normalized.includes("no such container") || + normalized.includes("no container with name or id") || + /container .+ does not exist/u.test(normalized) + ); +} + +function isMissingNetworkError(stderr: string): boolean { + const normalized = stderr.toLowerCase(); + return ( + normalized.includes("no such network") || + normalized.includes("network not found") || + /network .+ not found/u.test(normalized) || + /network .+ does not exist/u.test(normalized) + ); +} + +function validateNetworkName(networkName: string): string { + const normalized = networkName.trim(); + if (!normalized || normalized.startsWith("-")) { + throw new Error("Fleet network name is invalid."); + } + return normalized; +} + +export function createFleetContainerRuntime( + executor: FleetContainerCommandExecutor = defaultFleetContainerCommandExecutor, +) { + const execute = async ( + runtime: FleetContainerRuntimeName, + args: string[], + options: FleetContainerCommandOptions = {}, + ): Promise => { + try { + const result = await executor(runtime, args, options); + if (result.code !== 0 && !options.allowFailure) { + throw commandFailureError(runtime, args, result, options.redactValues); + } + return { + ...result, + stderr: redactEnvironmentValues(result.stderr, args, options.redactValues), + }; + } catch (error) { + throw formatExecutorError(error, runtime, args, options.redactValues); + } + }; + + return { + async assertLocal(runtime: FleetContainerRuntimeName): Promise { + if (runtime === "podman") { + const result = await execute("podman", ["info", "--format", "json"]); + if (parsePodmanServiceIsRemote(result.stdout)) { + throw new Error("Fleet requires local Podman; remote cells are not supported."); + } + return; + } + + // Docker exposes env-selected hosts as a virtual current context, including DOCKER_HOST. + const result = await execute("docker", ["context", "inspect"]); + const endpoint = parseDockerContextEndpoint(result.stdout); + if (!isLocalDockerEndpoint(endpoint)) { + throw new Error("Fleet requires a local Docker endpoint; remote cells are not supported."); + } + }, + + async inspect( + runtime: FleetContainerRuntimeName, + containerName: string, + ): Promise { + const args = ["container", "inspect", containerName]; + let result: FleetContainerCommandResult; + try { + result = await execute(runtime, args, { allowFailure: true }); + } catch (error) { + return { + kind: "unavailable", + state: "unknown", + error: formatExecutorError(error, runtime, args).message, + }; + } + if (result.code !== 0) { + if (isMissingContainerError(result.stderr)) { + return { kind: "missing", state: "missing" }; + } + return { + kind: "unavailable", + state: "unknown", + error: result.stderr.trim() || `${runtime} container inspect failed`, + }; + } + try { + return parseInspectOutput(result.stdout); + } catch { + return { + kind: "unavailable", + state: "unknown", + error: "container inspect returned an invalid response", + }; + } + }, + + async inspectNetwork( + runtime: FleetContainerRuntimeName, + networkName: string, + ): Promise { + const args = ["network", "inspect", validateNetworkName(networkName)]; + let result: FleetContainerCommandResult; + try { + result = await execute(runtime, args, { allowFailure: true }); + } catch (error) { + return { + kind: "unavailable", + error: formatExecutorError(error, runtime, args).message, + }; + } + if (result.code !== 0) { + if (isMissingNetworkError(result.stderr)) { + return { kind: "missing" }; + } + return { + kind: "unavailable", + error: result.stderr.trim() || `${runtime} network inspect failed`, + }; + } + try { + return parseNetworkInspectOutput(result.stdout); + } catch { + return { + kind: "unavailable", + error: "network inspect returned an invalid response", + }; + } + }, + + async isDockerRootless(): Promise { + const result = await execute("docker", ["info", "--format", "{{json .SecurityOptions}}"]); + return parseDockerRootlessInfo(result.stdout); + }, + + async run(profile: CellContainerProfile, start: boolean): Promise { + const tempRoot = await fs.realpath(os.tmpdir()); + const tempDir = await fs.mkdtemp(path.join(tempRoot, "openclaw-fleet-env-")); + const environmentFile = path.join(tempDir, "cell.env"); + try { + const args = start + ? buildCellRunArgs(profile, { environmentFile }) + : buildCellCreateArgs(profile, { environmentFile }); + const content = Object.entries(profile.environment) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => `${key}=${value}\n`) + .join(""); + await fs.writeFile(environmentFile, content, { encoding: "utf8", mode: 0o600 }); + await execute(profile.runtime, args, { + redactValues: Object.values(profile.environment), + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }, + + async pull(runtime: FleetContainerRuntimeName, image: string): Promise { + await execute(runtime, ["pull", validateFleetImage(image)]); + }, + + async createNetwork( + runtime: FleetContainerRuntimeName, + networkName: string, + labels: Readonly>, + ): Promise { + const labelArgs = Object.entries(labels) + .toSorted(([left], [right]) => left.localeCompare(right)) + .flatMap(([key, value]) => ["--label", `${key}=${value}`]); + await execute(runtime, [ + "network", + "create", + "--driver", + "bridge", + ...labelArgs, + validateNetworkName(networkName), + ]); + }, + + async removeNetwork(runtime: FleetContainerRuntimeName, networkName: string): Promise { + await execute(runtime, ["network", "rm", validateNetworkName(networkName)]); + }, + + async start(runtime: FleetContainerRuntimeName, containerName: string): Promise { + await execute(runtime, ["start", containerName]); + }, + + async stop(runtime: FleetContainerRuntimeName, containerName: string): Promise { + await execute(runtime, ["stop", containerName]); + }, + + async restart(runtime: FleetContainerRuntimeName, containerName: string): Promise { + await execute(runtime, ["restart", containerName]); + }, + + async remove( + runtime: FleetContainerRuntimeName, + containerName: string, + force: boolean, + ): Promise { + await execute(runtime, ["rm", ...(force ? ["--force"] : []), containerName]); + }, + }; +} diff --git a/src/fleet/registry.test.ts b/src/fleet/registry.test.ts new file mode 100644 index 000000000000..af77d9f572e3 --- /dev/null +++ b/src/fleet/registry.test.ts @@ -0,0 +1,166 @@ +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; +import { + acquireFleetCellOperation, + deleteFleetCell, + getFleetCell, + listFleetCells, + reserveFleetCell, + type ReserveFleetCellParams, + updateFleetCellImage, +} from "./registry.js"; + +describe("fleet cell registry", () => { + let root: string | undefined; + let env: NodeJS.ProcessEnv; + + const tempRoot = createSuiteTempRootTracker({ prefix: "openclaw-fleet-registry-" }); + + beforeEach(async () => { + root = await tempRoot.setup(); + env = { ...process.env, OPENCLAW_STATE_DIR: root }; + }); + + afterEach(async () => { + closeOpenClawStateDatabaseForTest(); + await tempRoot.cleanup(); + root = undefined; + }); + + function params(tenantId: string, requestedPort?: number): ReserveFleetCellParams { + if (!root) { + throw new Error("test root not initialized"); + } + return { + tenantId, + createdAtMs: 1, + image: "ghcr.io/openclaw/openclaw:latest", + runtime: "docker", + containerName: `openclaw-cell-${tenantId}`, + dataDir: path.join(root, "fleet", "cells", tenantId), + ...(requestedPort === undefined ? {} : { requestedPort }), + }; + } + + it("persists, orders, updates, and deletes cells", () => { + const zulu = reserveFleetCell(env, { + ...params("zulu", 19_250), + createdAtMs: 20, + runtime: "podman", + }); + const alpha = reserveFleetCell(env, { + ...params("alpha"), + createdAtMs: 10, + }); + + expect(alpha.hostPort).toBe(19_100); + expect(zulu.hostPort).toBe(19_250); + expect(listFleetCells(env).map((cell) => cell.tenantId)).toEqual(["alpha", "zulu"]); + expect(getFleetCell(env, "zulu")).toEqual(zulu); + + updateFleetCellImage(env, "zulu", "ghcr.io/openclaw/openclaw:v2"); + expect(getFleetCell(env, "zulu")?.image).toBe("ghcr.io/openclaw/openclaw:v2"); + + deleteFleetCell(env, "alpha"); + expect(getFleetCell(env, "alpha")).toBeUndefined(); + }); + + it("rejects duplicate tenant ids without replacing the row", () => { + const original = reserveFleetCell(env, params("alpha", 19_300)); + + expect(() => + reserveFleetCell(env, { + ...params("alpha", 19_301), + image: "ghcr.io/openclaw/openclaw:other", + }), + ).toThrow("Fleet cell already exists: alpha"); + expect(getFleetCell(env, "alpha")).toEqual(original); + }); + + it("allocates the first free port and rejects explicit collisions", () => { + expect(reserveFleetCell(env, params("alpha")).hostPort).toBe(19_100); + expect(reserveFleetCell(env, params("beta")).hostPort).toBe(19_101); + + expect(() => reserveFleetCell(env, params("gamma", 19_100))).toThrow(/19100/); + expect(getFleetCell(env, "gamma")).toBeUndefined(); + expect(reserveFleetCell(env, params("delta", 20_000)).hostPort).toBe(20_000); + }); + + it("serializes tenant mutations with renewable expiring leases", () => { + const first = acquireFleetCellOperation({ + env, + tenantId: "alpha", + operation: "upgrade", + owner: "first", + nowMs: 1_000, + }); + + expect(() => + acquireFleetCellOperation({ + env, + tenantId: "alpha", + operation: "rm", + owner: "second", + nowMs: 1_001, + }), + ).toThrow(/fleet upgrade.*already running/iu); + + first.heartbeat(200_000); + expect(() => + acquireFleetCellOperation({ + env, + tenantId: "alpha", + operation: "rm", + owner: "second", + nowMs: 400_000, + }), + ).toThrow(/already running/iu); + + first.release(); + const second = acquireFleetCellOperation({ + env, + tenantId: "alpha", + operation: "rm", + owner: "second", + nowMs: 400_001, + }); + second.release(); + }); + + it("fences an expired owner from its replacement lease", () => { + const first = acquireFleetCellOperation({ + env, + tenantId: "alpha", + operation: "upgrade", + owner: "first", + nowMs: 1_000, + }); + const successor = acquireFleetCellOperation({ + env, + tenantId: "alpha", + operation: "rm", + owner: "successor", + nowMs: 301_000, + }); + + expect(() => first.heartbeat(301_001)).toThrow(/lease was lost/iu); + first.release(); + expect(() => + acquireFleetCellOperation({ + env, + tenantId: "alpha", + operation: "create", + owner: "third", + nowMs: 301_002, + }), + ).toThrow(/fleet rm.*already running/iu); + + successor.release(); + }); + + it("fails image updates when the cell row disappeared", () => { + expect(() => updateFleetCellImage(env, "missing", "image:v2")).toThrow(/disappeared/iu); + }); +}); diff --git a/src/fleet/registry.ts b/src/fleet/registry.ts new file mode 100644 index 000000000000..25871d171980 --- /dev/null +++ b/src/fleet/registry.ts @@ -0,0 +1,282 @@ +import crypto from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { Insertable, Selectable } from "kysely"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; +import { allocateHostPort } from "./cell-profile.js"; + +export type FleetCellRecord = { + tenantId: string; + createdAtMs: number; + image: string; + runtime: "docker" | "podman"; + hostPort: number; + containerName: string; + dataDir: string; +}; + +export type ReserveFleetCellParams = Omit & { + requestedPort?: number; +}; + +type FleetCellsTable = OpenClawStateKyselyDatabase["fleet_cells"]; +type FleetCellRow = Selectable; +type FleetRegistryDatabase = Pick; + +const FLEET_OPERATION_LEASE_SCOPE = "fleet-cell-operation"; +const FLEET_OPERATION_LEASE_TTL_MS = 5 * 60_000; + +export type FleetCellOperationLease = { + heartbeat: (nowMs?: number) => void; + release: () => void; + owner: string; +}; + +export type FleetCellOperationName = "create" | "start" | "stop" | "restart" | "upgrade" | "rm"; + +function kyselyFor(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +function parseRuntime(runtime: string): FleetCellRecord["runtime"] { + if (runtime === "docker" || runtime === "podman") { + return runtime; + } + throw new Error(`Unsupported fleet runtime in state database: ${runtime}`); +} + +function rowToRecord(row: FleetCellRow): FleetCellRecord { + return { + tenantId: row.tenant_id, + createdAtMs: row.created_at_ms, + image: row.image, + runtime: parseRuntime(row.runtime), + hostPort: row.host_port, + containerName: row.container_name, + dataDir: row.data_dir, + }; +} + +function recordToRow(record: FleetCellRecord): Insertable { + return { + tenant_id: record.tenantId, + created_at_ms: record.createdAtMs, + image: record.image, + runtime: record.runtime, + host_port: record.hostPort, + container_name: record.containerName, + data_dir: record.dataDir, + }; +} + +export function listFleetCells(env: NodeJS.ProcessEnv = process.env): FleetCellRecord[] { + const db = openOpenClawStateDatabase({ env }).db; + const rows = executeSqliteQuerySync( + db, + kyselyFor(db).selectFrom("fleet_cells").selectAll().orderBy("tenant_id", "asc"), + ).rows; + return rows.map(rowToRecord); +} + +export function getFleetCell( + env: NodeJS.ProcessEnv, + tenantId: string, +): FleetCellRecord | undefined { + const db = openOpenClawStateDatabase({ env }).db; + const row = executeSqliteQueryTakeFirstSync( + db, + kyselyFor(db).selectFrom("fleet_cells").selectAll().where("tenant_id", "=", tenantId), + ); + return row ? rowToRecord(row) : undefined; +} + +export function reserveFleetCell( + env: NodeJS.ProcessEnv, + params: ReserveFleetCellParams, +): FleetCellRecord { + return runOpenClawStateWriteTransaction( + ({ db }) => { + const kysely = kyselyFor(db); + const existing = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("fleet_cells") + .select("tenant_id") + .where("tenant_id", "=", params.tenantId), + ); + if (existing) { + throw new Error(`Fleet cell already exists: ${params.tenantId}`); + } + + const usedPorts = executeSqliteQuerySync( + db, + kysely.selectFrom("fleet_cells").select("host_port"), + ).rows.map((row) => row.host_port); + // Allocate and reserve under one write lock so concurrent creates cannot claim one port. + const hostPort = allocateHostPort(usedPorts, params.requestedPort); + const record: FleetCellRecord = { + tenantId: params.tenantId, + createdAtMs: params.createdAtMs, + image: params.image, + runtime: params.runtime, + hostPort, + containerName: params.containerName, + dataDir: params.dataDir, + }; + executeSqliteQuerySync(db, kysely.insertInto("fleet_cells").values(recordToRow(record))); + return record; + }, + { env }, + ); +} + +export function updateFleetCellImage( + env: NodeJS.ProcessEnv, + tenantId: string, + image: string, +): void { + runOpenClawStateWriteTransaction( + ({ db }) => { + const result = executeSqliteQuerySync( + db, + kyselyFor(db).updateTable("fleet_cells").set({ image }).where("tenant_id", "=", tenantId), + ); + if (result.numAffectedRows !== 1n) { + throw new Error(`Fleet cell disappeared before its image could be updated: ${tenantId}`); + } + }, + { env }, + ); +} + +export function acquireFleetCellOperation(params: { + env: NodeJS.ProcessEnv; + tenantId: string; + operation: FleetCellOperationName; + owner?: string; + nowMs?: number; +}): FleetCellOperationLease { + const nowMs = params.nowMs ?? Date.now(); + const expiresAt = nowMs + FLEET_OPERATION_LEASE_TTL_MS; + const owner = params.owner ?? crypto.randomUUID(); + runOpenClawStateWriteTransaction( + ({ db }) => { + const kysely = kyselyFor(db); + executeSqliteQuerySync( + db, + kysely + .deleteFrom("state_leases") + .where("scope", "=", FLEET_OPERATION_LEASE_SCOPE) + .where("lease_key", "=", params.tenantId) + .where("expires_at", "<=", nowMs), + ); + const existing = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("state_leases") + .select(["expires_at", "payload_json"]) + .where("scope", "=", FLEET_OPERATION_LEASE_SCOPE) + .where("lease_key", "=", params.tenantId), + ); + if (existing) { + let operation = "fleet operation"; + try { + const payload: unknown = existing.payload_json + ? JSON.parse(existing.payload_json) + : undefined; + if ( + typeof payload === "object" && + payload !== null && + "operation" in payload && + typeof payload.operation === "string" + ) { + operation = `fleet ${payload.operation}`; + } + } catch { + // Busy diagnostics are best-effort; lease ownership remains authoritative. + } + throw new Error( + `Another ${operation} is already running for ${params.tenantId}; retry after ${new Date(existing.expires_at ?? expiresAt).toISOString()}.`, + ); + } + executeSqliteQuerySync( + db, + kysely.insertInto("state_leases").values({ + scope: FLEET_OPERATION_LEASE_SCOPE, + lease_key: params.tenantId, + owner, + expires_at: expiresAt, + heartbeat_at: nowMs, + payload_json: JSON.stringify({ operation: params.operation }), + created_at: nowMs, + updated_at: nowMs, + }), + ); + }, + { env: params.env }, + ); + + return { + owner, + heartbeat: (heartbeatNowMs = Date.now()) => { + const heartbeatExpiresAt = heartbeatNowMs + FLEET_OPERATION_LEASE_TTL_MS; + runOpenClawStateWriteTransaction( + ({ db }) => { + const result = executeSqliteQuerySync( + db, + kyselyFor(db) + .updateTable("state_leases") + .set({ + expires_at: heartbeatExpiresAt, + heartbeat_at: heartbeatNowMs, + updated_at: heartbeatNowMs, + }) + .where("scope", "=", FLEET_OPERATION_LEASE_SCOPE) + .where("lease_key", "=", params.tenantId) + .where("owner", "=", owner) + .where("expires_at", ">", heartbeatNowMs), + ); + if (result.numAffectedRows !== 1n) { + throw new Error(`Fleet operation lease was lost for ${params.tenantId}.`); + } + }, + { env: params.env }, + ); + }, + release: () => { + runOpenClawStateWriteTransaction( + ({ db }) => { + executeSqliteQuerySync( + db, + kyselyFor(db) + .deleteFrom("state_leases") + .where("scope", "=", FLEET_OPERATION_LEASE_SCOPE) + .where("lease_key", "=", params.tenantId) + .where("owner", "=", owner), + ); + }, + { env: params.env }, + ); + }, + }; +} + +export function deleteFleetCell(env: NodeJS.ProcessEnv, tenantId: string): void { + runOpenClawStateWriteTransaction( + ({ db }) => { + executeSqliteQuerySync( + db, + kyselyFor(db).deleteFrom("fleet_cells").where("tenant_id", "=", tenantId), + ); + }, + { env }, + ); +} diff --git a/src/fleet/service-removal.runtime.test.ts b/src/fleet/service-removal.runtime.test.ts new file mode 100644 index 000000000000..9e69ce0769a0 --- /dev/null +++ b/src/fleet/service-removal.runtime.test.ts @@ -0,0 +1,356 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; +import { cellAuthSecretDir, cellOwnerId } from "./cell-profile.js"; +import type { FleetContainerInspectResult, FleetContainerRuntime } from "./containers.runtime.js"; +import { getFleetCell, reserveFleetCell } from "./registry.js"; +import { createFleetService } from "./service.runtime.js"; + +let root: string; +const TEST_ATTEMPT_ID = "22222222222222222222222222222222"; + +function fleetLabels(tenant = "acme", attemptId = TEST_ATTEMPT_ID): Record { + return { + "openclaw.fleet.tenant": tenant, + "openclaw.fleet.owner": cellOwnerId(path.join(root, "fleet", "cells", tenant)), + "openclaw.fleet.attempt": attemptId, + "openclaw.fleet.env-keys": "FEATURE", + }; +} + +function runningInspection( + overrides: Partial> = {}, +): Extract { + return { + kind: "ok", + state: "running", + running: true, + labels: fleetLabels(), + environment: { + HOME: "/home/node", + OPENCLAW_GATEWAY_TOKEN: "old-token", + FEATURE: "enabled", + NODE_VERSION: "old-image-default", + }, + imageId: "sha256:old-image-id", + memory: "2147483648", + cpus: "2", + pidsLimit: 512, + ...overrides, + }; +} + +function createContainerMock( + initialInspection: FleetContainerInspectResult = { + kind: "missing", + state: "missing", + }, +) { + const assertLocal = vi.fn(async () => undefined); + const inspect = vi.fn(async () => initialInspection); + const networks = new Map< + string, + Extract>, { kind: "ok" }> + >(); + const inspectNetwork = vi.fn( + async (_runtime, name) => networks.get(name) ?? { kind: "missing" }, + ); + const isDockerRootless = vi.fn(async () => false); + const run = vi.fn(async () => undefined); + const pull = vi.fn(async () => undefined); + const createNetwork = vi.fn( + async (_runtime, name, labels) => { + networks.set(name, { + kind: "ok", + labels: { ...labels }, + attachedContainers: [], + }); + }, + ); + const removeNetwork = vi.fn(async (_runtime, name) => { + networks.delete(name); + }); + const start = vi.fn(async () => undefined); + const stop = vi.fn(async () => undefined); + const restart = vi.fn(async () => undefined); + const remove = vi.fn(async () => { + inspect.mockResolvedValue({ kind: "missing", state: "missing" }); + }); + return { + runtime: { + assertLocal, + inspect, + inspectNetwork, + isDockerRootless, + run, + pull, + createNetwork, + removeNetwork, + start, + stop, + restart, + remove, + }, + assertLocal, + inspect, + inspectNetwork, + isDockerRootless, + run, + pull, + createNetwork, + removeNetwork, + start, + stop, + restart, + remove, + }; +} + +describe("fleet service filesystem and removal", () => { + let env: NodeJS.ProcessEnv; + + const tempRoot = createSuiteTempRootTracker({ prefix: "openclaw-fleet-service-" }); + + beforeEach(async () => { + root = await tempRoot.setup(); + env = { ...process.env, OPENCLAW_STATE_DIR: root }; + }); + + afterEach(async () => { + closeOpenClawStateDatabaseForTest(); + await tempRoot.cleanup(); + }); + + it("maps rootless users and SELinux mounts for Docker and Podman", async () => { + const docker = createContainerMock(); + docker.isDockerRootless.mockResolvedValue(true); + await createFleetService({ + env, + containers: docker.runtime, + getuid: () => 1001, + getgid: () => 1002, + selinuxEnabled: async () => true, + }).create({ tenant: "docker-cell", gatewayToken: "token" }); + expect(docker.run.mock.calls[0]?.[0].containerUser).toEqual({ + mode: "numeric", + uid: 0, + gid: 0, + }); + expect(docker.run.mock.calls[0]?.[0].selinuxRelabel).toBe(true); + + const podman = createContainerMock(); + await createFleetService({ + env, + containers: podman.runtime, + getuid: () => 1001, + getgid: () => 1002, + selinuxEnabled: async () => true, + }).create({ tenant: "podman-cell", runtime: "podman", gatewayToken: "token" }); + expect(podman.isDockerRootless).not.toHaveBeenCalled(); + expect(podman.run.mock.calls[0]?.[0]).toMatchObject({ + containerUser: { mode: "podman-keep-id", uid: 1001, gid: 1002 }, + selinuxRelabel: true, + }); + }); + + it("refuses a realpath escape before container or registry removal", async () => { + const containers = createContainerMock(); + const cellsRoot = path.join(root, "fleet", "cells"); + const outside = path.join(root, "outside-cell"); + await fs.mkdir(cellsRoot, { recursive: true }); + await fs.mkdir(outside); + reserveFleetCell(env, { + tenantId: "escape", + createdAtMs: 1000, + image: "ghcr.io/openclaw/openclaw:latest", + runtime: "docker", + containerName: "openclaw-cell-escape", + dataDir: outside, + }); + const service = createFleetService({ env, containers: containers.runtime }); + + await expect( + service.remove({ tenant: "escape", purgeData: true, force: true }), + ).rejects.toThrow(/outside its fleet-owned directory/iu); + expect(containers.inspect).not.toHaveBeenCalled(); + expect(getFleetCell(env, "escape")).toBeDefined(); + await expect(fs.stat(outside)).resolves.toBeDefined(); + }); + + it("refuses to purge a tenant symlinked to a sibling cell", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + await service.create({ tenant: "beta", gatewayToken: "token" }); + const acmeDir = path.join(root, "fleet", "cells", "acme"); + const betaDir = path.join(root, "fleet", "cells", "beta"); + await fs.rm(acmeDir, { recursive: true }); + await fs.symlink(betaDir, acmeDir, "dir"); + + await expect(service.remove({ tenant: "acme", purgeData: true, force: true })).rejects.toThrow( + /symlinked fleet tenant directory/iu, + ); + + expect(containers.inspect).not.toHaveBeenCalled(); + await expect(fs.stat(path.join(betaDir, "openclaw.json"))).resolves.toBeDefined(); + expect(getFleetCell(env, "acme")).toBeDefined(); + }); + + it("refuses a tenant-controlled config symlink during recreate", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + await service.create({ tenant: "beta", gatewayToken: "token" }); + await service.remove({ tenant: "acme" }); + const acmeConfig = path.join(root, "fleet", "cells", "acme", "openclaw.json"); + const betaConfig = path.join(root, "fleet", "cells", "beta", "openclaw.json"); + const betaBefore = await fs.readFile(betaConfig, "utf8"); + await fs.rm(acmeConfig); + await fs.symlink("../beta/openclaw.json", acmeConfig); + const runCount = containers.run.mock.calls.length; + containers.inspect.mockResolvedValue(runningInspection({ state: "created", running: false })); + const retryService = createFleetService({ + env, + containers: containers.runtime, + now: () => 1000, + generateAttemptId: () => TEST_ATTEMPT_ID, + }); + + await expect(retryService.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /unsafe cell config/iu, + ); + + expect(containers.run).toHaveBeenCalledTimes(runCount + 1); + expect(containers.remove).toHaveBeenCalledWith("docker", "openclaw-cell-acme", true); + expect(getFleetCell(env, "acme")).toBeUndefined(); + await expect(fs.readFile(betaConfig, "utf8")).resolves.toBe(betaBefore); + }); + + it("validates rejected recreate input before rewriting retained config", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + await service.remove({ tenant: "acme" }); + const configPath = path.join(root, "fleet", "cells", "acme", "openclaw.json"); + const configBefore = await fs.readFile(configPath, "utf8"); + const runCount = containers.run.mock.calls.length; + + await expect( + service.create({ tenant: "acme", gatewayToken: "token", env: ["INVALID"] }), + ).rejects.toThrow(/expected KEY=VAL/iu); + + expect(containers.run).toHaveBeenCalledTimes(runCount); + expect(getFleetCell(env, "acme")).toBeUndefined(); + await expect(fs.readFile(configPath, "utf8")).resolves.toBe(configBefore); + }); + + it("purges a contained cell only after forced container removal", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection()); + + await expect(service.remove({ tenant: "acme", purgeData: true, force: true })).resolves.toEqual( + { tenant: "acme", action: "rm", dataPurged: true }, + ); + + expect(containers.remove).toHaveBeenCalledWith("docker", "openclaw-cell-acme", true); + expect(containers.removeNetwork).toHaveBeenCalledWith("docker", "openclaw-cell-acme-net"); + expect(getFleetCell(env, "acme")).toBeUndefined(); + await expect(fs.stat(path.join(root, "fleet", "cells", "acme"))).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(fs.stat(cellAuthSecretDir(root, "acme"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("passes explicit force through even when inspect observed a stopped container", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection({ state: "exited", running: false })); + + await service.remove({ tenant: "acme", force: true }); + + expect(containers.remove).toHaveBeenCalledWith("docker", "openclaw-cell-acme", true); + }); + + it("retains state when network removal fails and completes on retry", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection({ state: "exited", running: false })); + containers.removeNetwork.mockRejectedValueOnce(new Error("network still in use")); + + await expect(service.remove({ tenant: "acme", purgeData: true, force: true })).rejects.toThrow( + /still in use/iu, + ); + expect(getFleetCell(env, "acme")).toBeDefined(); + await expect(fs.stat(path.join(root, "fleet", "cells", "acme"))).resolves.toBeDefined(); + + await expect( + service.remove({ tenant: "acme", purgeData: true, force: true }), + ).resolves.toMatchObject({ tenant: "acme", dataPurged: true }); + expect(getFleetCell(env, "acme")).toBeUndefined(); + }); + + it("finishes purge when one exact tenant directory is already missing", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + await fs.rm(path.join(root, "fleet", "cells", "acme"), { recursive: true }); + + await expect(service.remove({ tenant: "acme", purgeData: true, force: true })).resolves.toEqual( + { tenant: "acme", action: "rm", dataPurged: true }, + ); + + expect(getFleetCell(env, "acme")).toBeUndefined(); + await expect(fs.stat(cellAuthSecretDir(root, "acme"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("refuses to remove a container when its cell network belongs to another profile", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection({ state: "exited", running: false })); + containers.inspectNetwork.mockResolvedValue({ + kind: "ok", + labels: { + ...fleetLabels(), + "openclaw.fleet.owner": "11111111111111111111111111111111", + }, + attachedContainers: [], + }); + containers.remove.mockClear(); + + await expect(service.remove({ tenant: "acme" })).rejects.toThrow( + /acme-net.*ownership labels/iu, + ); + expect(containers.remove).not.toHaveBeenCalled(); + expect(containers.removeNetwork).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeDefined(); + }); + + it("refuses removal before mutation when an unexpected network peer is attached", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection({ state: "exited", running: false })); + containers.inspectNetwork.mockResolvedValue({ + kind: "ok", + labels: fleetLabels(), + attachedContainers: [{ id: "peer-id", name: "unexpected-peer" }], + }); + containers.remove.mockClear(); + + await expect(service.remove({ tenant: "acme" })).rejects.toThrow(/unexpected containers/iu); + expect(containers.remove).not.toHaveBeenCalled(); + expect(containers.removeNetwork).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeDefined(); + }); +}); diff --git a/src/fleet/service-support.runtime.ts b/src/fleet/service-support.runtime.ts new file mode 100644 index 000000000000..98b88026446c --- /dev/null +++ b/src/fleet/service-support.runtime.ts @@ -0,0 +1,573 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import JSON5 from "json5"; +import { FsSafeError, root as fsSafeRoot } from "../infra/fs-safe.js"; +import { replaceFileAtomic } from "../infra/replace-file.js"; +import { isRecord } from "../utils.js"; +import { + buildCellEnvironment, + cellNetworkName, + cellOwnerId, + FLEET_ATTEMPT_LABEL, + FLEET_ENV_KEYS_LABEL, + FLEET_OWNER_LABEL, + FLEET_TENANT_LABEL, + parseEnvAssignments, + validateTenantId, + type CellContainerProfile, + type FleetContainerRuntimeName, +} from "./cell-profile.js"; +import type { + FleetContainerInspectResult, + FleetContainerRuntime, + FleetNetworkInspectResult, +} from "./containers.runtime.js"; +import { + acquireFleetCellOperation, + getFleetCell, + type FleetCellOperationName, + type FleetCellRecord, +} from "./registry.js"; + +const CELL_CONFIG_FILENAME = "openclaw.json"; +const HEALTH_TIMEOUT_MS = 1_000; +const CELL_CONFIG_MAX_BYTES = 4 * 1024 * 1024; +const FLEET_OPERATION_HEARTBEAT_MS = 60_000; + +export type FleetHealthResult = + | { status: "ok"; url: string; httpStatus: number } + | { status: "failed"; url: string; error: string; httpStatus?: number } + | { status: "skipped"; url: string; reason: string }; + +function requiredRecord(value: unknown, label: string): Record { + if (!isRecord(value)) { + throw new Error(`${label} must be an object.`); + } + return value; +} + +function optionalRecord(value: unknown, label: string): Record { + if (value === undefined) { + return {}; + } + return requiredRecord(value, label); +} + +function readAllowedOrigins(value: unknown): string[] { + if (value === undefined) { + return []; + } + if (!Array.isArray(value) || !value.every((origin) => typeof origin === "string")) { + throw new Error("gateway.controlUi.allowedOrigins must be an array of strings."); + } + return value; +} + +async function ensurePrivateDirectory(dir: string): Promise { + await fs.mkdir(dir, { recursive: true, mode: 0o700 }); + const stat = await fs.lstat(dir); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new Error(`Refusing to use non-directory fleet data path: ${dir}`); + } + await fs.chmod(dir, 0o700); +} + +export async function prepareCellDirectories( + record: FleetCellRecord, + authSecretDir: string, + owner?: { uid: number; gid: number }, +): Promise { + await Promise.all([ + ensurePrivateDirectory(record.dataDir), + ensurePrivateDirectory(authSecretDir), + ]); + if (owner) { + await Promise.all([ + fs.chown(record.dataDir, owner.uid, owner.gid), + fs.chown(authSecretDir, owner.uid, owner.gid), + ]); + } +} + +export async function prepareCellConfig( + record: FleetCellRecord, + owner?: { uid: number; gid: number }, +): Promise { + const configPath = path.join(record.dataDir, CELL_CONFIG_FILENAME); + let rootConfig: Record; + const cellRoot = await fsSafeRoot(record.dataDir, { + hardlinks: "reject", + maxBytes: CELL_CONFIG_MAX_BYTES, + nonBlockingRead: true, + symlinks: "reject", + }); + try { + const read = await cellRoot.read(CELL_CONFIG_FILENAME); + rootConfig = requiredRecord(JSON5.parse(read.buffer.toString("utf8")), "Cell config"); + } catch (error) { + if (error instanceof FsSafeError && error.code === "not-found") { + rootConfig = {}; + } else if (error instanceof FsSafeError) { + throw new Error(`Refusing to read unsafe cell config: ${configPath}`, { cause: error }); + } else { + throw error; + } + } + + const gateway = optionalRecord(rootConfig.gateway, "gateway"); + const auth = optionalRecord(gateway.auth, "gateway.auth"); + const controlUi = optionalRecord(gateway.controlUi, "gateway.controlUi"); + const nextAuth: Record = { ...auth, mode: "token" }; + delete nextAuth.token; + const origins = new Set(readAllowedOrigins(controlUi.allowedOrigins)); + origins.add(`http://localhost:${record.hostPort}`); + origins.add(`http://127.0.0.1:${record.hostPort}`); + + const nextConfig = { + ...rootConfig, + gateway: { + ...gateway, + mode: "local", + bind: "lan", + auth: nextAuth, + controlUi: { + ...controlUi, + allowedOrigins: [...origins], + }, + }, + }; + await replaceFileAtomic({ + filePath: configPath, + content: `${JSON.stringify(nextConfig, null, 2)}\n`, + dirMode: 0o700, + mode: 0o600, + tempPrefix: CELL_CONFIG_FILENAME, + copyFallbackOnPermissionError: true, + }); + if (owner) { + await fs.chown(configPath, owner.uid, owner.gid); + } +} + +export type HostIdentity = { uid: number; gid: number }; + +export function readHostIdentity( + getuid: () => number | undefined, + getgid: () => number | undefined, +): HostIdentity | undefined { + const uid = getuid(); + const gid = getgid(); + if (uid === undefined || gid === undefined) { + return undefined; + } + if (!Number.isSafeInteger(uid) || uid < 0 || !Number.isSafeInteger(gid) || gid < 0) { + throw new Error("Host uid and gid must be non-negative integers."); + } + return { uid, gid }; +} + +export async function resolveContainerUser(params: { + runtime: FleetContainerRuntimeName; + containers: FleetContainerRuntime; + hostIdentity: HostIdentity | undefined; + user?: string; +}): Promise { + const match = params.user?.match(/^(\d+):(\d+)$/u); + if (match) { + const uid = Number(match[1]); + const gid = Number(match[2]); + return params.runtime === "podman" + ? { mode: "podman-keep-id", uid, gid } + : { mode: "numeric", uid, gid }; + } + if (!params.hostIdentity) { + return undefined; + } + if (params.runtime === "podman") { + return params.hostIdentity.uid === 0 + ? undefined + : { mode: "podman-keep-id", ...params.hostIdentity }; + } + if (await params.containers.isDockerRootless()) { + // Root in a rootless Docker user namespace maps to the invoking host user, not host root. + return { mode: "numeric", uid: 0, gid: 0 }; + } + return params.hostIdentity.uid === 0 ? undefined : { mode: "numeric", ...params.hostIdentity }; +} + +export async function detectHostSelinux(): Promise { + if (process.platform !== "linux") { + return false; + } + try { + await fs.access("/sys/fs/selinux"); + return true; + } catch { + return false; + } +} + +export function inspectionState( + record: FleetCellRecord, + inspection: FleetContainerInspectResult, +): string { + if (inspection.kind !== "ok") { + return inspection.state; + } + return inspection.labels[FLEET_TENANT_LABEL] === record.tenantId && + inspection.labels[FLEET_OWNER_LABEL] === cellOwnerId(record.dataDir) + ? inspection.state + : "unknown"; +} + +export function assertManagedInspection( + record: FleetCellRecord, + inspection: FleetContainerInspectResult, +): Extract { + if (inspection.kind === "missing") { + throw new Error(`Fleet container is missing for tenant ${record.tenantId}.`); + } + if (inspection.kind === "unavailable") { + throw new Error( + `Cannot inspect ${record.runtime} container for tenant ${record.tenantId}: ${inspection.error}`, + ); + } + if ( + inspection.labels[FLEET_TENANT_LABEL] !== record.tenantId || + inspection.labels[FLEET_OWNER_LABEL] !== cellOwnerId(record.dataDir) + ) { + throw new Error( + `Refusing to manage ${record.containerName}: fleet ownership labels do not match tenant ${record.tenantId}.`, + ); + } + return inspection; +} + +export async function probeCellHealth(params: { + port: number; + fetchImpl: typeof fetch; +}): Promise { + const url = `http://127.0.0.1:${params.port}/healthz`; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), HEALTH_TIMEOUT_MS); + let response: Response | undefined; + try { + response = await params.fetchImpl(url, { + method: "GET", + redirect: "manual", + signal: controller.signal, + }); + if (response.ok) { + return { status: "ok", url, httpStatus: response.status }; + } + return { + status: "failed", + url, + httpStatus: response.status, + error: `HTTP ${response.status}`, + }; + } catch (error) { + return { + status: "failed", + url, + error: error instanceof Error ? error.message : String(error), + }; + } finally { + clearTimeout(timeout); + await response?.body?.cancel().catch(() => undefined); + } +} + +export async function resolvePurgeTarget( + rootDir: string, + targetDir: string, + tenantId: string, +): Promise { + const expectedTarget = path.resolve(rootDir, tenantId); + if (path.resolve(targetDir) !== expectedTarget) { + throw new Error(`Refusing to purge data outside its fleet-owned directory: ${targetDir}`); + } + let targetStat: Awaited>; + try { + targetStat = await fs.lstat(expectedTarget); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } + if (targetStat.isSymbolicLink()) { + throw new Error(`Refusing to purge a symlinked fleet tenant directory: ${targetDir}`); + } + const root = await fs.realpath(rootDir); + const target = await fs.realpath(targetDir); + const relative = path.relative(root, target); + // Require the exact real tenant leaf; an in-root sibling symlink could otherwise delete another cell. + if ( + target !== path.join(root, tenantId) || + !relative || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new Error(`Refusing to purge data outside its fleet-owned directory: ${targetDir}`); + } + return target; +} + +export function requireCell(env: NodeJS.ProcessEnv, tenant: string): FleetCellRecord { + const tenantId = validateTenantId(tenant); + const record = getFleetCell(env, tenantId); + if (!record) { + throw new Error(`Fleet cell not found: ${tenantId}`); + } + return record; +} + +export function assertCurrentReservation(env: NodeJS.ProcessEnv, expected: FleetCellRecord): void { + const current = getFleetCell(env, expected.tenantId); + if ( + !current || + current.createdAtMs !== expected.createdAtMs || + current.image !== expected.image || + current.runtime !== expected.runtime || + current.hostPort !== expected.hostPort || + current.containerName !== expected.containerName || + current.dataDir !== expected.dataDir + ) { + throw new Error(`Fleet create reservation changed while provisioning ${expected.tenantId}.`); + } +} + +export function requirePositiveResource(value: string, label: string): string { + const parsed = Number(value); + if (!value.trim() || !Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Cannot upgrade cell: inspected ${label} limit is missing or invalid.`); + } + return value; +} + +export function requirePidsLimit(value: number | undefined): number { + if (value === undefined || !Number.isSafeInteger(value) || value < 1) { + throw new Error("Cannot upgrade cell: inspected PID limit is missing or invalid."); + } + return value; +} + +export function rebuildInspectedEnvironment( + environment: Readonly>, + labels: Readonly>, + token: string, +): Record { + const encodedKeys = labels[FLEET_ENV_KEYS_LABEL]; + if (encodedKeys === undefined) { + throw new Error("Cannot upgrade cell: user environment provenance label is missing."); + } + const keys = encodedKeys ? encodedKeys.split(",") : []; + if (new Set(keys).size !== keys.length || keys.toSorted().join(",") !== encodedKeys) { + throw new Error("Cannot upgrade cell: user environment provenance label is invalid."); + } + const assignments = keys.map((key) => { + const value = environment[key]; + if (value === undefined) { + throw new Error(`Cannot upgrade cell: inspected environment is missing ${key}.`); + } + return `${key}=${value}`; + }); + return buildCellEnvironment(token, parseEnvAssignments(assignments)); +} + +export async function cleanupFailedCreateContainer( + record: FleetCellRecord, + containers: FleetContainerRuntime, + attemptId: string, + checkpoint: () => void, +): Promise { + const inspection = await containers.inspect(record.runtime, record.containerName); + if (inspection.kind === "missing") { + return true; + } + if (inspection.kind === "unavailable") { + return false; + } + const tenantLabel = inspection.labels[FLEET_TENANT_LABEL]; + const ownerLabel = inspection.labels[FLEET_OWNER_LABEL]; + // Fleet always labels what it creates, so a container without fleet labels (or with + // another owner's labels) is foreign: leave it untouched but release the reservation, + // otherwise a name collision strands a tenant no fleet command can recover. + if (tenantLabel !== record.tenantId || ownerLabel !== cellOwnerId(record.dataDir)) { + return true; + } + if (inspection.labels[FLEET_ATTEMPT_LABEL] !== attemptId) { + return false; + } + checkpoint(); + await containers.remove(record.runtime, record.containerName, true); + return (await containers.inspect(record.runtime, record.containerName)).kind === "missing"; +} + +export async function cleanupFailedCreateNetwork( + record: FleetCellRecord, + containers: FleetContainerRuntime, + attemptId: string, + checkpoint: () => void, +): Promise { + const networkName = cellNetworkName(record.tenantId); + const inspection = await containers.inspectNetwork(record.runtime, networkName); + if (inspection.kind === "missing") { + return true; + } + if (inspection.kind === "unavailable") { + return false; + } + const tenantLabel = inspection.labels[FLEET_TENANT_LABEL]; + const ownerLabel = inspection.labels[FLEET_OWNER_LABEL]; + // Same foreign-resource rule as cleanupFailedCreateContainer: unlabeled or + // other-owner networks are never fleet's to delete, but must not pin the reservation. + if (tenantLabel !== record.tenantId || ownerLabel !== cellOwnerId(record.dataDir)) { + return true; + } + if ( + inspection.labels[FLEET_ATTEMPT_LABEL] !== attemptId || + inspection.attachedContainers.length > 0 + ) { + return false; + } + checkpoint(); + await containers.removeNetwork(record.runtime, networkName); + return (await containers.inspectNetwork(record.runtime, networkName)).kind === "missing"; +} + +function inspectionHasFleetOwner( + record: FleetCellRecord, + inspection: Extract, +): boolean { + return ( + inspection.labels[FLEET_TENANT_LABEL] === record.tenantId && + inspection.labels[FLEET_OWNER_LABEL] === cellOwnerId(record.dataDir) + ); +} + +export function assertManagedNetwork( + record: FleetCellRecord, + inspection: FleetNetworkInspectResult, +): Extract { + if (inspection.kind === "missing") { + throw new Error(`Fleet network is missing for tenant ${record.tenantId}.`); + } + if (inspection.kind === "unavailable") { + throw new Error( + `Cannot inspect ${record.runtime} network for tenant ${record.tenantId}: ${inspection.error}`, + ); + } + if ( + inspection.labels[FLEET_TENANT_LABEL] !== record.tenantId || + inspection.labels[FLEET_OWNER_LABEL] !== cellOwnerId(record.dataDir) + ) { + throw new Error( + `Refusing to manage ${cellNetworkName(record.tenantId)}: fleet ownership labels do not match tenant ${record.tenantId}.`, + ); + } + const unexpectedAttachments = inspection.attachedContainers.filter( + (container) => container.name !== record.containerName, + ); + if (unexpectedAttachments.length > 0 || inspection.attachedContainers.length > 1) { + throw new Error( + `Refusing to manage ${cellNetworkName(record.tenantId)}: unexpected containers are attached.`, + ); + } + return inspection; +} + +export async function restorePreviousCell(params: { + record: FleetCellRecord; + containers: FleetContainerRuntime; + oldProfile: CellContainerProfile; + previousAttemptId: string; + nextAttemptId: string; + wasRunning: boolean; + checkpoint: () => void; +}): Promise { + const current = await params.containers.inspect( + params.record.runtime, + params.record.containerName, + ); + if (current.kind === "unavailable") { + throw new Error(current.error); + } + if (current.kind === "ok") { + if (!inspectionHasFleetOwner(params.record, current)) { + throw new Error("container ownership changed during upgrade recovery"); + } + const currentAttemptId = current.labels[FLEET_ATTEMPT_LABEL]; + if (currentAttemptId === params.previousAttemptId) { + if (current.running !== params.wasRunning) { + params.checkpoint(); + await params.containers[current.running ? "stop" : "start"]( + params.record.runtime, + params.record.containerName, + ); + } + return; + } + if (currentAttemptId !== params.nextAttemptId) { + throw new Error("container generation changed during upgrade recovery"); + } + params.checkpoint(); + await params.containers.remove(params.record.runtime, params.record.containerName, true); + } + params.checkpoint(); + await params.containers.run(params.oldProfile, params.wasRunning); +} + +export async function withFleetCellOperation(params: { + env: NodeJS.ProcessEnv; + tenantId: string; + operationName: FleetCellOperationName; + operation: (checkpoint: () => void) => Promise; +}): Promise { + const lease = acquireFleetCellOperation({ + env: params.env, + tenantId: params.tenantId, + operation: params.operationName, + }); + let heartbeatError: unknown; + const checkpoint = () => { + try { + lease.heartbeat(); + heartbeatError = undefined; + } catch (error) { + heartbeatError = error; + throw error; + } + }; + const heartbeat = setInterval(() => { + try { + lease.heartbeat(); + heartbeatError = undefined; + } catch (error) { + heartbeatError = error; + } + }, FLEET_OPERATION_HEARTBEAT_MS); + heartbeat.unref(); + let result: T; + try { + result = await params.operation(checkpoint); + if (heartbeatError) { + checkpoint(); + } else { + lease.heartbeat(); + } + } catch (error) { + clearInterval(heartbeat); + try { + lease.release(); + } catch { + // Preserve the operation or fencing error; a release failure is secondary. + } + throw error; + } + clearInterval(heartbeat); + lease.release(); + return result; +} diff --git a/src/fleet/service.runtime.test.ts b/src/fleet/service.runtime.test.ts new file mode 100644 index 000000000000..433e50d72ee8 --- /dev/null +++ b/src/fleet/service.runtime.test.ts @@ -0,0 +1,759 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; +import { cellAuthSecretDir, cellOwnerId } from "./cell-profile.js"; +import type { FleetContainerInspectResult, FleetContainerRuntime } from "./containers.runtime.js"; +import { deleteFleetCell, getFleetCell } from "./registry.js"; +import { createFleetService } from "./service.runtime.js"; + +let root: string; +const TEST_ATTEMPT_ID = "22222222222222222222222222222222"; +const NEXT_ATTEMPT_ID = "44444444444444444444444444444444"; + +function fleetLabels(tenant = "acme", attemptId = TEST_ATTEMPT_ID): Record { + return { + "openclaw.fleet.tenant": tenant, + "openclaw.fleet.owner": cellOwnerId(path.join(root, "fleet", "cells", tenant)), + "openclaw.fleet.attempt": attemptId, + "openclaw.fleet.env-keys": "FEATURE", + }; +} + +function runningInspection( + overrides: Partial> = {}, +): Extract { + return { + kind: "ok", + state: "running", + running: true, + labels: fleetLabels(), + environment: { + HOME: "/home/node", + OPENCLAW_GATEWAY_TOKEN: "old-token", + FEATURE: "enabled", + NODE_VERSION: "old-image-default", + }, + imageId: "sha256:old-image-id", + memory: "2147483648", + cpus: "2", + pidsLimit: 512, + ...overrides, + }; +} + +function createContainerMock( + initialInspection: FleetContainerInspectResult = { + kind: "missing", + state: "missing", + }, +) { + const assertLocal = vi.fn(async () => undefined); + const inspect = vi.fn(async () => initialInspection); + const networks = new Map< + string, + Extract>, { kind: "ok" }> + >(); + const inspectNetwork = vi.fn( + async (_runtime, name) => networks.get(name) ?? { kind: "missing" }, + ); + const isDockerRootless = vi.fn(async () => false); + const run = vi.fn(async () => undefined); + const pull = vi.fn(async () => undefined); + const createNetwork = vi.fn( + async (_runtime, name, labels) => { + networks.set(name, { + kind: "ok", + labels: { ...labels }, + attachedContainers: [], + }); + }, + ); + const removeNetwork = vi.fn(async (_runtime, name) => { + networks.delete(name); + }); + const start = vi.fn(async () => undefined); + const stop = vi.fn(async () => undefined); + const restart = vi.fn(async () => undefined); + const remove = vi.fn(async () => { + inspect.mockResolvedValue({ kind: "missing", state: "missing" }); + }); + return { + runtime: { + assertLocal, + inspect, + inspectNetwork, + isDockerRootless, + run, + pull, + createNetwork, + removeNetwork, + start, + stop, + restart, + remove, + }, + assertLocal, + inspect, + inspectNetwork, + isDockerRootless, + run, + pull, + createNetwork, + removeNetwork, + start, + stop, + restart, + remove, + }; +} + +describe("fleet service", () => { + let env: NodeJS.ProcessEnv; + + const tempRoot = createSuiteTempRootTracker({ prefix: "openclaw-fleet-service-" }); + + beforeEach(async () => { + root = await tempRoot.setup(); + env = { ...process.env, OPENCLAW_STATE_DIR: root }; + }); + + afterEach(async () => { + closeOpenClawStateDatabaseForTest(); + await tempRoot.cleanup(); + }); + + it("creates a bootable token-only cell config and returns the secret-bearing result", async () => { + const containers = createContainerMock(); + const service = createFleetService({ + env, + containers: containers.runtime, + now: () => 1_700_000_000_000, + generateToken: () => "gw-token", + }); + + const result = await service.create({ + tenant: "acme", + env: ["FEATURE=a=b"], + }); + + expect(result).toEqual({ + tenant: "acme", + containerName: "openclaw-cell-acme", + port: 19_100, + image: "ghcr.io/openclaw/openclaw:latest", + runtime: "docker", + started: true, + token: "gw-token", + tokenNote: "Shown once. Store this Gateway token securely.", + url: "http://127.0.0.1:19100", + nextStep: + "Open http://127.0.0.1:19100, then configure per-tenant channel accounts inside the cell.", + }); + expect(containers.run).toHaveBeenCalledOnce(); + const [profile, start] = containers.run.mock.calls[0] ?? []; + expect(start).toBe(false); + expect(containers.createNetwork).toHaveBeenCalledWith("docker", "openclaw-cell-acme-net", { + "openclaw.fleet.tenant": "acme", + "openclaw.fleet.owner": cellOwnerId(path.join(root, "fleet", "cells", "acme")), + "openclaw.fleet.attempt": expect.stringMatching(/^[a-f0-9]{32}$/u), + }); + expect(containers.start).toHaveBeenCalledWith("docker", "openclaw-cell-acme"); + expect(profile?.networkName).toBe("openclaw-cell-acme-net"); + expect(containers.createNetwork.mock.invocationCallOrder[0] ?? -1).toBeLessThan( + containers.run.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(containers.run.mock.invocationCallOrder[0] ?? -1).toBeLessThan( + containers.start.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(profile?.environment).toMatchObject({ + OPENCLAW_GATEWAY_TOKEN: "gw-token", + FEATURE: "a=b", + }); + + const dataDir = path.join(root, "fleet", "cells", "acme"); + const config = JSON.parse(await fs.readFile(path.join(dataDir, "openclaw.json"), "utf8")) as { + gateway?: { + mode?: string; + bind?: string; + auth?: Record; + controlUi?: { allowedOrigins?: string[] }; + }; + }; + expect(config.gateway).toMatchObject({ + mode: "local", + bind: "lan", + auth: { mode: "token" }, + controlUi: { + allowedOrigins: ["http://localhost:19100", "http://127.0.0.1:19100"], + }, + }); + expect(config.gateway?.auth).not.toHaveProperty("token"); + const authSecretDir = cellAuthSecretDir(root, "acme"); + await expect(fs.stat(authSecretDir)).resolves.toBeDefined(); + expect(path.relative(dataDir, authSecretDir)).toMatch(/^\.\./u); + }); + + it("generates a 32-character hexadecimal token", async () => { + const containers = createContainerMock(); + const result = await createFleetService({ env, containers: containers.runtime }).create({ + tenant: "random-token", + start: false, + }); + + expect(result.token).toMatch(/^[a-f0-9]{32}$/u); + }); + + it("rejects a remote runtime before registry or filesystem mutation", async () => { + const containers = createContainerMock(); + containers.assertLocal.mockRejectedValue( + new Error("Fleet requires a local Docker endpoint; remote cells are not supported."), + ); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + await expect(service.create({ tenant: "remote", gatewayToken: "token" })).rejects.toThrow( + /local Docker endpoint.*remote cells/iu, + ); + + expect(getFleetCell(env, "remote")).toBeUndefined(); + expect(containers.createNetwork).not.toHaveBeenCalled(); + expect(containers.run).not.toHaveBeenCalled(); + await expect(fs.stat(path.join(root, "fleet", "cells", "remote"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("lists cells deterministically and degrades runtime failures to unknown", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "zulu", gatewayToken: "z-token" }); + await service.create({ tenant: "alpha", gatewayToken: "a-token" }); + containers.inspect.mockImplementation(async (_runtime, name) => + name.endsWith("alpha") + ? runningInspection({ labels: fleetLabels("alpha") }) + : { kind: "unavailable", state: "unknown", error: "daemon unavailable" }, + ); + + const cells = await service.list(); + + expect(cells.map((cell) => [cell.tenant, cell.state])).toEqual([ + ["alpha", "running"], + ["zulu", "unknown"], + ]); + expect(JSON.stringify(cells)).not.toContain("old-token"); + }); + + it("reports live status with a bounded loopback health probe", async () => { + const containers = createContainerMock(); + const fetchMock = vi.fn(async () => new Response(null, { status: 200 })); + const service = createFleetService({ + env, + containers: containers.runtime, + fetch: fetchMock, + now: () => 1000, + }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection()); + + const status = await service.status("acme"); + + expect(status.container).toEqual({ state: "running", running: true, managed: true }); + expect(status.health).toEqual({ + status: "ok", + url: "http://127.0.0.1:19100/healthz", + httpStatus: 200, + }); + expect(fetchMock).toHaveBeenCalledWith( + "http://127.0.0.1:19100/healthz", + expect.objectContaining({ method: "GET", redirect: "manual" }), + ); + expect(JSON.stringify(status)).not.toContain("old-token"); + }); + + it("reports failed and skipped health outcomes without probing a stopped cell", async () => { + const containers = createContainerMock(); + const fetchMock = vi.fn(async () => new Response(null, { status: 503 })); + const service = createFleetService({ + env, + containers: containers.runtime, + fetch: fetchMock, + now: () => 1000, + }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection({ state: "exited", running: false })); + + await expect(service.status("acme")).resolves.toMatchObject({ + health: { status: "skipped", reason: "container is not running" }, + }); + expect(fetchMock).not.toHaveBeenCalled(); + + containers.inspect.mockResolvedValue(runningInspection()); + await expect(service.status("acme")).resolves.toMatchObject({ + health: { status: "failed", httpStatus: 503, error: "HTTP 503" }, + }); + }); + + it.each(["start", "stop", "restart"] as const)("runs the %s lifecycle action", async (action) => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection()); + containers[action].mockClear(); + + await expect(service.lifecycle("acme", action)).resolves.toEqual({ tenant: "acme", action }); + + expect(containers[action]).toHaveBeenCalledWith("docker", "openclaw-cell-acme"); + }); + + it("carries inspected environment and resources through upgrade", async () => { + const containers = createContainerMock(); + const service = createFleetService({ + env, + containers: containers.runtime, + fetch: vi.fn(async () => new Response(null, { status: 200 })), + now: () => 1000, + generateAttemptId: () => NEXT_ATTEMPT_ID, + }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.run.mockClear(); + containers.inspect + .mockResolvedValue(runningInspection()) + .mockResolvedValueOnce(runningInspection()) + .mockResolvedValueOnce(runningInspection({ labels: fleetLabels("acme", NEXT_ATTEMPT_ID) })); + + const result = await service.upgrade("acme", "ghcr.io/openclaw/openclaw:v2"); + + expect(result).toEqual({ + tenant: "acme", + action: "upgrade", + image: "ghcr.io/openclaw/openclaw:v2", + }); + expect(containers.pull).toHaveBeenCalledWith("docker", "ghcr.io/openclaw/openclaw:v2"); + expect(containers.stop).toHaveBeenCalledWith("docker", "openclaw-cell-acme"); + expect(containers.remove).toHaveBeenCalledWith("docker", "openclaw-cell-acme", false); + expect(containers.inspectNetwork).toHaveBeenCalledWith("docker", "openclaw-cell-acme-net"); + const [profile, start] = containers.run.mock.calls[0] ?? []; + expect(start).toBe(true); + expect(profile).toMatchObject({ + image: "ghcr.io/openclaw/openclaw:v2", + hostPort: 19_100, + memory: "2147483648", + cpus: "2", + pidsLimit: 512, + networkName: "openclaw-cell-acme-net", + environment: { + HOME: "/home/node", + OPENCLAW_GATEWAY_TOKEN: "old-token", + FEATURE: "enabled", + }, + }); + expect(profile?.environment).not.toHaveProperty("NODE_VERSION"); + expect(getFleetCell(env, "acme")?.image).toBe("ghcr.io/openclaw/openclaw:v2"); + }); + + it("restores the immutable old image when replacement fails", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.run.mockClear(); + containers.inspect + .mockResolvedValueOnce(runningInspection()) + .mockResolvedValueOnce({ kind: "missing", state: "missing" }); + containers.run.mockRejectedValueOnce(new Error("replacement failed")).mockResolvedValueOnce(); + + await expect(service.upgrade("acme")).rejects.toThrow(/previous container was restored/iu); + + expect(containers.run).toHaveBeenCalledTimes(2); + expect(containers.run.mock.calls[1]?.[0].image).toBe("sha256:old-image-id"); + expect(getFleetCell(env, "acme")?.image).toBe("ghcr.io/openclaw/openclaw:latest"); + }); + + it("restarts the old cell when removal fails after stop", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.run.mockClear(); + containers.start.mockClear(); + containers.inspect + .mockResolvedValueOnce(runningInspection()) + .mockResolvedValueOnce(runningInspection({ state: "exited", running: false })); + containers.remove.mockRejectedValueOnce(new Error("daemon busy")); + + await expect(service.upgrade("acme")).rejects.toThrow(/previous container was restored/iu); + + expect(containers.stop).toHaveBeenCalledWith("docker", "openclaw-cell-acme"); + expect(containers.start).toHaveBeenCalledWith("docker", "openclaw-cell-acme"); + expect(containers.run).not.toHaveBeenCalled(); + }); + + it("restores the old cell when the replacement registry update fails", async () => { + const containers = createContainerMock(); + const service = createFleetService({ + env, + containers: containers.runtime, + fetch: vi.fn(async () => new Response(null, { status: 200 })), + now: () => 1000, + generateAttemptId: () => NEXT_ATTEMPT_ID, + updateImage: () => { + throw new Error("state database is full"); + }, + }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.run.mockClear(); + containers.remove.mockClear(); + containers.inspect + .mockResolvedValueOnce(runningInspection()) + .mockResolvedValueOnce(runningInspection({ labels: fleetLabels("acme", NEXT_ATTEMPT_ID) })) + .mockResolvedValueOnce(runningInspection({ labels: fleetLabels("acme", NEXT_ATTEMPT_ID) })); + + await expect(service.upgrade("acme")).rejects.toThrow(/previous container was restored/iu); + + expect(containers.run).toHaveBeenCalledTimes(2); + expect(containers.run.mock.calls[1]?.[0].image).toBe("sha256:old-image-id"); + expect(containers.remove).toHaveBeenCalledWith("docker", "openclaw-cell-acme", true); + expect(containers.removeNetwork).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")?.image).toBe("ghcr.io/openclaw/openclaw:latest"); + }); + + it("restores the previous cell when the replacement container is not running", async () => { + const containers = createContainerMock(); + const service = createFleetService({ + env, + containers: containers.runtime, + now: () => 1000, + generateAttemptId: () => NEXT_ATTEMPT_ID, + }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.run.mockClear(); + const crashLooping = runningInspection({ + labels: fleetLabels("acme", NEXT_ATTEMPT_ID), + state: "restarting", + running: false, + }); + containers.inspect + .mockResolvedValueOnce(runningInspection()) + .mockResolvedValueOnce(crashLooping) + .mockResolvedValueOnce(crashLooping); + + await expect(service.upgrade("acme")).rejects.toThrow(/previous container was restored/iu); + + expect(containers.run).toHaveBeenCalledTimes(2); + expect(containers.run.mock.calls[1]?.[0].image).toBe("sha256:old-image-id"); + expect(getFleetCell(env, "acme")?.image).toBe("ghcr.io/openclaw/openclaw:latest"); + }); + + it("restores the previous cell when the replacement crashes after starting", async () => { + const containers = createContainerMock(); + const service = createFleetService({ + env, + containers: containers.runtime, + fetch: vi.fn(async () => { + throw new Error("connect ECONNREFUSED"); + }), + sleep: async () => {}, + now: () => 1000, + generateAttemptId: () => NEXT_ATTEMPT_ID, + }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.run.mockClear(); + const crashed = runningInspection({ + labels: fleetLabels("acme", NEXT_ATTEMPT_ID), + state: "exited", + running: false, + }); + containers.inspect + .mockResolvedValueOnce(runningInspection()) + .mockResolvedValueOnce(runningInspection({ labels: fleetLabels("acme", NEXT_ATTEMPT_ID) })) + .mockResolvedValueOnce(crashed) + .mockResolvedValueOnce(crashed); + + await expect(service.upgrade("acme")).rejects.toThrow(/previous container was restored/iu); + + expect(containers.run).toHaveBeenCalledTimes(2); + expect(containers.run.mock.calls[1]?.[0].image).toBe("sha256:old-image-id"); + expect(getFleetCell(env, "acme")?.image).toBe("ghcr.io/openclaw/openclaw:latest"); + }); + + it("restores the previous cell when the replacement never becomes healthy", async () => { + const containers = createContainerMock(); + let clock = 0; + const service = createFleetService({ + env, + containers: containers.runtime, + fetch: vi.fn(async () => new Response(null, { status: 503 })), + sleep: async () => {}, + now: () => (clock += 50_000), + generateAttemptId: () => NEXT_ATTEMPT_ID, + }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.run.mockClear(); + const hung = runningInspection({ labels: fleetLabels("acme", NEXT_ATTEMPT_ID) }); + containers.inspect + .mockResolvedValueOnce(runningInspection()) + .mockResolvedValueOnce(hung) + .mockResolvedValueOnce(hung) + .mockResolvedValueOnce(hung); + + await expect(service.upgrade("acme")).rejects.toThrow(/previous container was restored/iu); + + expect(containers.run).toHaveBeenCalledTimes(2); + expect(containers.run.mock.calls[1]?.[0].image).toBe("sha256:old-image-id"); + expect(getFleetCell(env, "acme")?.image).toBe("ghcr.io/openclaw/openclaw:latest"); + }); + + it("refuses upgrade before pull or removal when the inspected token is missing", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.inspect.mockResolvedValue( + runningInspection({ environment: { HOME: "/home/node" } }), + ); + + await expect(service.upgrade("acme")).rejects.toThrow(/no Gateway token environment/iu); + expect(containers.pull).not.toHaveBeenCalled(); + expect(containers.stop).not.toHaveBeenCalled(); + expect(containers.remove).not.toHaveBeenCalled(); + }); + + it("refuses upgrade when an unexpected container is attached to the cell network", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "old-token" }); + containers.inspect.mockResolvedValue(runningInspection()); + containers.inspectNetwork.mockResolvedValue({ + kind: "ok", + labels: fleetLabels(), + attachedContainers: [ + { id: "cell-id", name: "openclaw-cell-acme" }, + { id: "peer-id", name: "unexpected-peer" }, + ], + }); + + await expect(service.upgrade("acme")).rejects.toThrow(/unexpected containers/iu); + expect(containers.pull).toHaveBeenCalledOnce(); + expect(containers.stop).not.toHaveBeenCalled(); + expect(containers.remove).not.toHaveBeenCalled(); + }); + + it("rejects option-like images before create or upgrade mutations", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + await expect( + service.create({ tenant: "bad-image", image: "--help", gatewayToken: "token" }), + ).rejects.toThrow(/image must not begin/iu); + expect(getFleetCell(env, "bad-image")).toBeUndefined(); + expect(containers.run).not.toHaveBeenCalled(); + + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection()); + await expect(service.upgrade("acme", "--help")).rejects.toThrow(/image must not begin/iu); + expect(containers.pull).not.toHaveBeenCalled(); + expect(containers.stop).not.toHaveBeenCalled(); + expect(containers.remove).not.toHaveBeenCalled(); + }); + + it("requires force for running removal and purge", async () => { + const containers = createContainerMock(); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + await service.create({ tenant: "acme", gatewayToken: "token" }); + containers.inspect.mockResolvedValue(runningInspection()); + + await expect(service.remove({ tenant: "acme" })).rejects.toThrow(/running.*--force/iu); + await expect(service.remove({ tenant: "acme", purgeData: true })).rejects.toThrow( + "--purge-data requires --force.", + ); + expect(containers.remove).not.toHaveBeenCalled(); + }); + + it("removes a labeled partial container before releasing a failed create", async () => { + const containers = createContainerMock(runningInspection()); + containers.run.mockRejectedValue(new Error("host port is already allocated")); + const service = createFleetService({ + env, + containers: containers.runtime, + now: () => 1000, + generateAttemptId: () => TEST_ATTEMPT_ID, + }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /already allocated/iu, + ); + + expect(containers.remove).toHaveBeenCalledWith("docker", "openclaw-cell-acme", true); + expect(getFleetCell(env, "acme")).toBeUndefined(); + }); + + it("retains a failed-create reservation when partial cleanup is uncertain", async () => { + const containers = createContainerMock({ + kind: "unavailable", + state: "unknown", + error: "daemon unavailable", + }); + containers.run.mockRejectedValue(new Error("container command timed out")); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /timed out/iu, + ); + + expect(containers.remove).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeDefined(); + }); + + it("cleans up its exact-attempt network when network creation fails", async () => { + const containers = createContainerMock(); + containers.createNetwork.mockRejectedValue(new Error("network create timed out")); + containers.inspectNetwork + .mockResolvedValueOnce({ + kind: "ok", + labels: fleetLabels(), + attachedContainers: [], + }) + .mockResolvedValueOnce({ kind: "missing" }); + const service = createFleetService({ + env, + containers: containers.runtime, + now: () => 1000, + generateAttemptId: () => TEST_ATTEMPT_ID, + }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /timed out/iu, + ); + + expect(containers.run).not.toHaveBeenCalled(); + expect(containers.removeNetwork).toHaveBeenCalledWith("docker", "openclaw-cell-acme-net"); + expect(getFleetCell(env, "acme")).toBeUndefined(); + }); + + it("serializes same-tenant mutations across service instances", async () => { + const containers = createContainerMock(); + let releaseNetwork: (() => void) | undefined; + containers.createNetwork.mockImplementation( + async () => + await new Promise((resolve) => { + releaseNetwork = resolve; + }), + ); + const first = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + const second = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + const creating = first.create({ tenant: "acme", gatewayToken: "token" }); + await vi.waitFor(() => expect(containers.createNetwork).toHaveBeenCalledOnce()); + await expect(second.create({ tenant: "acme", gatewayToken: "other-token" })).rejects.toThrow( + /fleet create.*already running/iu, + ); + + releaseNetwork?.(); + await expect(creating).resolves.toMatchObject({ tenant: "acme" }); + }); + + it("releases a failed operation lease for a retry", async () => { + const containers = createContainerMock(); + containers.createNetwork.mockRejectedValueOnce(new Error("daemon busy")); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /daemon busy/iu, + ); + await expect( + service.create({ tenant: "acme", gatewayToken: "retry-token" }), + ).resolves.toMatchObject({ tenant: "acme" }); + }); + + it("removes its exact-attempt container when the reservation disappears mid-create", async () => { + const containers = createContainerMock(runningInspection({ state: "created", running: false })); + containers.run.mockImplementation(async () => { + deleteFleetCell(env, "acme"); + }); + const service = createFleetService({ + env, + containers: containers.runtime, + now: () => 1000, + generateAttemptId: () => TEST_ATTEMPT_ID, + }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /reservation changed/iu, + ); + + expect(containers.remove).toHaveBeenCalledWith("docker", "openclaw-cell-acme", true); + expect(containers.start).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeUndefined(); + }); + + it("releases the reservation when an unlabeled foreign container holds the cell name", async () => { + const containers = createContainerMock(runningInspection({ labels: {} })); + containers.run.mockRejectedValue(new Error("container name is already in use")); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /already in use/iu, + ); + + expect(containers.remove).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeUndefined(); + }); + + it("releases the reservation when an unlabeled foreign network holds the cell name", async () => { + const containers = createContainerMock(); + containers.createNetwork.mockRejectedValue(new Error("network name is already in use")); + containers.inspectNetwork.mockResolvedValue({ + kind: "ok", + labels: {}, + attachedContainers: [], + }); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /already in use/iu, + ); + + expect(containers.removeNetwork).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeUndefined(); + }); + + it("never removes a same-tenant container owned by another profile", async () => { + const containers = createContainerMock( + runningInspection({ + labels: { + ...fleetLabels(), + "openclaw.fleet.owner": "11111111111111111111111111111111", + }, + }), + ); + containers.run.mockRejectedValue(new Error("container name is already in use")); + const service = createFleetService({ env, containers: containers.runtime, now: () => 1000 }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /already in use/iu, + ); + + expect(containers.remove).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeUndefined(); + }); + + it("never removes a same-profile container that predates the create attempt", async () => { + const containers = createContainerMock( + runningInspection({ + labels: fleetLabels("acme", "33333333333333333333333333333333"), + }), + ); + containers.run.mockRejectedValue(new Error("container name is already in use")); + const service = createFleetService({ + env, + containers: containers.runtime, + now: () => 1000, + generateAttemptId: () => TEST_ATTEMPT_ID, + }); + + await expect(service.create({ tenant: "acme", gatewayToken: "token" })).rejects.toThrow( + /already in use/iu, + ); + + expect(containers.remove).not.toHaveBeenCalled(); + expect(getFleetCell(env, "acme")).toBeDefined(); + }); +}); diff --git a/src/fleet/service.runtime.ts b/src/fleet/service.runtime.ts new file mode 100644 index 000000000000..f86a199bebba --- /dev/null +++ b/src/fleet/service.runtime.ts @@ -0,0 +1,636 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { resolveStateDir } from "../config/paths.js"; +import { + buildCellEnvironment, + cellAuthSecretDir, + cellContainerName, + cellDataDir, + cellNetworkName, + cellOwnerId, + DEFAULT_FLEET_IMAGE, + FLEET_ATTEMPT_LABEL, + FLEET_OWNER_LABEL, + FLEET_TENANT_LABEL, + parseEnvAssignments, + validateCellContainerProfile, + validateFleetImage, + validateTenantId, + type CellContainerProfile, + type FleetContainerRuntimeName, +} from "./cell-profile.js"; +import { + createFleetContainerRuntime, + type FleetContainerInspectResult, + type FleetContainerRuntime, +} from "./containers.runtime.js"; +import { + deleteFleetCell, + listFleetCells, + reserveFleetCell, + updateFleetCellImage, +} from "./registry.js"; +import { + assertCurrentReservation, + assertManagedInspection, + assertManagedNetwork, + cleanupFailedCreateContainer, + cleanupFailedCreateNetwork, + detectHostSelinux, + inspectionState, + prepareCellConfig, + prepareCellDirectories, + probeCellHealth, + readHostIdentity, + rebuildInspectedEnvironment, + requireCell, + requirePidsLimit, + requirePositiveResource, + resolveContainerUser, + resolvePurgeTarget, + restorePreviousCell, + withFleetCellOperation, +} from "./service-support.runtime.js"; + +const OFFICIAL_IMAGE_UID = 1_000; +const OFFICIAL_IMAGE_GID = 1_000; +// Mirrors the compose healthcheck contract: an upgrade commits only after /healthz +// answers. The deadline bounds how long a broken image can hold the cell before +// restore without rolling back slow-booting cells prematurely. +const UPGRADE_VERIFY_TIMEOUT_MS = 60_000; +const UPGRADE_VERIFY_POLL_MS = 1_000; + +export type FleetCreateOptions = { + tenant: string; + image?: string; + runtime?: FleetContainerRuntimeName; + port?: number; + memory?: string; + cpus?: string; + pidsLimit?: number; + env?: string[]; + gatewayToken?: string; + start?: boolean; +}; + +export type FleetCreateResult = { + tenant: string; + containerName: string; + port: number; + image: string; + runtime: FleetContainerRuntimeName; + started: boolean; + token: string; + tokenNote: string; + url: string; + nextStep: string; +}; + +export type FleetListEntry = { + tenant: string; + state: string; + port: number; + image: string; + created: string; +}; + +export type FleetHealthResult = + | { status: "ok"; url: string; httpStatus: number } + | { status: "failed"; url: string; error: string; httpStatus?: number } + | { status: "skipped"; url: string; reason: string }; + +export type FleetStatusResult = { + tenant: string; + containerName: string; + runtime: FleetContainerRuntimeName; + port: number; + image: string; + created: string; + dataDir: string; + container: + | { state: string; running: boolean; managed: boolean } + | { state: "missing"; running: false; managed: false } + | { state: "unknown"; running: false; managed: false; error: string }; + health: FleetHealthResult; +}; + +export type FleetLifecycleAction = "start" | "stop" | "restart"; + +export type FleetActionResult = { + tenant: string; + action: FleetLifecycleAction | "upgrade" | "rm"; + image?: string; + dataPurged?: boolean; +}; + +export type FleetServiceOptions = { + env?: NodeJS.ProcessEnv; + containers?: FleetContainerRuntime; + fetch?: typeof fetch; + now?: () => number; + generateToken?: () => string; + generateAttemptId?: () => string; + getuid?: () => number | undefined; + getgid?: () => number | undefined; + sleep?: (ms: number) => Promise; + selinuxEnabled?: () => Promise; + updateImage?: typeof updateFleetCellImage; +}; + +export function createFleetService(options: FleetServiceOptions = {}) { + const env = options.env ?? process.env; + const containers = options.containers ?? createFleetContainerRuntime(); + const fetchImpl = options.fetch ?? fetch; + const now = options.now ?? Date.now; + const { generateToken = () => crypto.randomBytes(16).toString("hex") } = options; + const generateAttemptId = + options.generateAttemptId ?? (() => crypto.randomBytes(16).toString("hex")); + const getuid = options.getuid ?? (() => process.getuid?.()); + const getgid = options.getgid ?? (() => process.getgid?.()); + const sleep = + options.sleep ?? + ((ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + })); + const selinuxEnabled = options.selinuxEnabled ?? detectHostSelinux; + const updateImage = options.updateImage ?? updateFleetCellImage; + + return { + async create(createOptions: FleetCreateOptions): Promise { + const tenantId = validateTenantId(createOptions.tenant); + const image = validateFleetImage(createOptions.image ?? DEFAULT_FLEET_IMAGE); + const runtime = createOptions.runtime ?? "docker"; + const { gatewayToken } = createOptions; + if (gatewayToken !== undefined && !gatewayToken.trim()) { + throw new Error("Gateway token must not be empty."); + } + const token = gatewayToken ?? generateToken(); + const environment = buildCellEnvironment(token, parseEnvAssignments(createOptions.env ?? [])); + const attemptId = generateAttemptId(); + await containers.assertLocal(runtime); + return await withFleetCellOperation({ + env, + tenantId, + operationName: "create", + operation: async (checkpoint) => { + checkpoint(); + const stateDir = resolveStateDir(env); + const record = reserveFleetCell(env, { + tenantId, + createdAtMs: now(), + image, + runtime, + requestedPort: createOptions.port, + containerName: cellContainerName(tenantId), + dataDir: cellDataDir(stateDir, tenantId), + }); + + let networkAttempted = false; + let containerAttempted = false; + try { + const authSecretDir = cellAuthSecretDir(stateDir, tenantId); + const hostIdentity = readHostIdentity(getuid, getgid); + const containerUser = await resolveContainerUser({ + runtime, + containers, + hostIdentity, + }); + const imageOwner = + hostIdentity?.uid === 0 && !containerUser + ? { uid: OFFICIAL_IMAGE_UID, gid: OFFICIAL_IMAGE_GID } + : undefined; + const profile: CellContainerProfile = { + tenantId, + containerName: record.containerName, + networkName: cellNetworkName(tenantId), + image, + runtime, + hostPort: record.hostPort, + dataDir: record.dataDir, + authSecretDir, + ownerId: cellOwnerId(record.dataDir), + attemptId, + memory: createOptions.memory ?? "2g", + cpus: createOptions.cpus ?? "2", + pidsLimit: createOptions.pidsLimit ?? 512, + environment, + ...(containerUser ? { containerUser } : {}), + selinuxRelabel: await selinuxEnabled(), + }; + validateCellContainerProfile(profile); + checkpoint(); + await prepareCellDirectories(record, authSecretDir, imageOwner); + assertCurrentReservation(env, record); + const started = createOptions.start !== false; + networkAttempted = true; + checkpoint(); + await containers.createNetwork(runtime, profile.networkName, { + [FLEET_TENANT_LABEL]: tenantId, + [FLEET_OWNER_LABEL]: profile.ownerId, + [FLEET_ATTEMPT_LABEL]: attemptId, + }); + assertCurrentReservation(env, record); + containerAttempted = true; + checkpoint(); + await containers.run(profile, false); + assertCurrentReservation(env, record); + checkpoint(); + await prepareCellConfig(record, imageOwner); + assertCurrentReservation(env, record); + if (started) { + checkpoint(); + await containers.start(runtime, record.containerName); + assertCurrentReservation(env, record); + } + const url = `http://127.0.0.1:${record.hostPort}`; + return { + tenant: tenantId, + containerName: record.containerName, + port: record.hostPort, + image, + runtime, + started, + token, + tokenNote: "Shown once. Store this Gateway token securely.", + url, + nextStep: `Open ${url}, then configure per-tenant channel accounts inside the cell.`, + }; + } catch (error) { + let releaseReservation = true; + try { + if (containerAttempted) { + releaseReservation = await cleanupFailedCreateContainer( + record, + containers, + attemptId, + checkpoint, + ); + } + if (releaseReservation && networkAttempted) { + releaseReservation = await cleanupFailedCreateNetwork( + record, + containers, + attemptId, + checkpoint, + ); + } + } catch { + releaseReservation = false; + } + if (releaseReservation) { + try { + checkpoint(); + deleteFleetCell(env, tenantId); + } catch { + // Preserve the provisioning error; a stale reservation remains recoverable via fleet list/rm. + } + } + throw error; + } + }, + }); + }, + + async list(): Promise { + const records = listFleetCells(env); + const localityChecks = new Map>(); + const inspections = await Promise.all( + records.map(async (record) => { + try { + let locality = localityChecks.get(record.runtime); + if (!locality) { + locality = containers.assertLocal(record.runtime); + localityChecks.set(record.runtime, locality); + } + await locality; + return await containers.inspect(record.runtime, record.containerName); + } catch (error) { + return { + kind: "unavailable" as const, + state: "unknown" as const, + error: error instanceof Error ? error.message : String(error), + }; + } + }), + ); + return records.map((record, index) => ({ + tenant: record.tenantId, + state: inspectionState( + record, + inspections[index] ?? { + kind: "unavailable", + state: "unknown", + error: "inspect result missing", + }, + ), + port: record.hostPort, + image: record.image, + created: new Date(record.createdAtMs).toISOString(), + })); + }, + + async status(tenant: string): Promise { + const record = requireCell(env, tenant); + let inspection: FleetContainerInspectResult; + try { + await containers.assertLocal(record.runtime); + inspection = await containers.inspect(record.runtime, record.containerName); + } catch (error) { + inspection = { + kind: "unavailable" as const, + state: "unknown" as const, + error: error instanceof Error ? error.message : String(error), + }; + } + const url = `http://127.0.0.1:${record.hostPort}/healthz`; + let container: FleetStatusResult["container"]; + let health: FleetHealthResult; + if (inspection.kind === "ok") { + const managed = + inspection.labels[FLEET_TENANT_LABEL] === record.tenantId && + inspection.labels[FLEET_OWNER_LABEL] === cellOwnerId(record.dataDir); + container = { + state: managed ? inspection.state : "unknown", + running: inspection.running, + managed, + }; + health = + managed && inspection.running + ? await probeCellHealth({ port: record.hostPort, fetchImpl }) + : { + status: "skipped", + url, + reason: managed ? "container is not running" : "fleet ownership mismatch", + }; + } else if (inspection.kind === "missing") { + container = { state: "missing", running: false, managed: false }; + health = { status: "skipped", url, reason: "container is missing" }; + } else { + container = { state: "unknown", running: false, managed: false, error: inspection.error }; + health = { status: "skipped", url, reason: "container runtime unavailable" }; + } + return { + tenant: record.tenantId, + containerName: record.containerName, + runtime: record.runtime, + port: record.hostPort, + image: record.image, + created: new Date(record.createdAtMs).toISOString(), + dataDir: record.dataDir, + container, + health, + }; + }, + + async lifecycle(tenant: string, action: FleetLifecycleAction): Promise { + const tenantId = validateTenantId(tenant); + await containers.assertLocal(requireCell(env, tenantId).runtime); + return await withFleetCellOperation({ + env, + tenantId, + operationName: action, + operation: async (checkpoint) => { + const record = requireCell(env, tenantId); + await containers.assertLocal(record.runtime); + assertManagedInspection( + record, + await containers.inspect(record.runtime, record.containerName), + ); + checkpoint(); + await containers[action](record.runtime, record.containerName); + return { tenant: record.tenantId, action }; + }, + }); + }, + + async upgrade(tenant: string, requestedImage?: string): Promise { + const tenantId = validateTenantId(tenant); + const explicitImage = + requestedImage === undefined ? undefined : validateFleetImage(requestedImage); + await containers.assertLocal(requireCell(env, tenantId).runtime); + return await withFleetCellOperation({ + env, + tenantId, + operationName: "upgrade", + operation: async (checkpoint) => { + const record = requireCell(env, tenantId); + await containers.assertLocal(record.runtime); + const inspection = assertManagedInspection( + record, + await containers.inspect(record.runtime, record.containerName), + ); + const image = explicitImage ?? validateFleetImage(record.image); + // The token is intentionally absent from SQLite; capture container env before removal and replay it. + const { OPENCLAW_GATEWAY_TOKEN: token } = inspection.environment; + if (!token) { + throw new Error( + "Cannot upgrade cell: existing container has no Gateway token environment.", + ); + } + const containerUser = await resolveContainerUser({ + runtime: record.runtime, + containers, + hostIdentity: readHostIdentity(getuid, getgid), + user: inspection.user, + }); + const previousAttemptId = inspection.labels[FLEET_ATTEMPT_LABEL]; + if (!previousAttemptId || !/^[a-f0-9]{32}$/u.test(previousAttemptId)) { + throw new Error("Cannot upgrade cell: container attempt label is missing or invalid."); + } + const nextAttemptId = generateAttemptId(); + const profileBase = { + tenantId: record.tenantId, + containerName: record.containerName, + networkName: cellNetworkName(record.tenantId), + runtime: record.runtime, + hostPort: record.hostPort, + dataDir: record.dataDir, + authSecretDir: cellAuthSecretDir(resolveStateDir(env), record.tenantId), + ownerId: cellOwnerId(record.dataDir), + memory: requirePositiveResource(inspection.memory, "memory"), + cpus: requirePositiveResource(inspection.cpus, "CPU"), + pidsLimit: requirePidsLimit(inspection.pidsLimit), + environment: rebuildInspectedEnvironment( + inspection.environment, + inspection.labels, + token, + ), + ...(containerUser ? { containerUser } : {}), + selinuxRelabel: await selinuxEnabled(), + } satisfies Omit; + const oldProfile: CellContainerProfile = { + ...profileBase, + image: inspection.imageId, + attemptId: previousAttemptId, + }; + const nextProfile: CellContainerProfile = { + ...profileBase, + image, + attemptId: nextAttemptId, + }; + validateCellContainerProfile(oldProfile); + validateCellContainerProfile(nextProfile); + + checkpoint(); + await containers.pull(record.runtime, image); + checkpoint(); + assertManagedNetwork( + record, + await containers.inspectNetwork(record.runtime, cellNetworkName(record.tenantId)), + ); + try { + if (inspection.running) { + checkpoint(); + await containers.stop(record.runtime, record.containerName); + } + checkpoint(); + await containers.remove(record.runtime, record.containerName, false); + checkpoint(); + await containers.run(nextProfile, true); + // `run -d` succeeds once the container launches, and a broken image can stay + // "running" briefly before crashing. Commit only after the replacement answers + // /healthz (the image's compose health contract): exit/restart-loop fails fast, + // and the deadline restores the old cell instead of leaving a dead replacement. + const verifyDeadline = now() + UPGRADE_VERIFY_TIMEOUT_MS; + for (;;) { + const replacement = await containers.inspect(record.runtime, record.containerName); + if ( + replacement.kind !== "ok" || + replacement.labels[FLEET_ATTEMPT_LABEL] !== nextAttemptId || + !replacement.running + ) { + throw new Error( + replacement.kind === "ok" + ? "Replacement cell container is not running after upgrade." + : "Replacement cell container could not be verified after upgrade.", + ); + } + const health = await probeCellHealth({ port: record.hostPort, fetchImpl }); + if (health.status === "ok") { + break; + } + if (now() >= verifyDeadline) { + throw new Error("Replacement cell container did not become healthy after upgrade."); + } + checkpoint(); + await sleep(UPGRADE_VERIFY_POLL_MS); + } + checkpoint(); + updateImage(env, record.tenantId, image); + } catch (error) { + try { + await restorePreviousCell({ + record, + containers, + oldProfile, + previousAttemptId, + nextAttemptId, + wasRunning: inspection.running, + checkpoint, + }); + } catch { + throw new Error( + `Fleet upgrade failed for ${record.tenantId}; the previous container could not be restored.`, + { cause: error }, + ); + } + throw new Error( + `Fleet upgrade failed for ${record.tenantId}; the previous container was restored.`, + { cause: error }, + ); + } + return { tenant: record.tenantId, action: "upgrade", image }; + }, + }); + }, + + async remove(params: { + tenant: string; + force?: boolean; + purgeData?: boolean; + }): Promise { + if (params.purgeData && !params.force) { + throw new Error("--purge-data requires --force."); + } + const tenantId = validateTenantId(params.tenant); + await containers.assertLocal(requireCell(env, tenantId).runtime); + return await withFleetCellOperation({ + env, + tenantId, + operationName: "rm", + operation: async (checkpoint) => { + const record = requireCell(env, tenantId); + await containers.assertLocal(record.runtime); + const stateDir = resolveStateDir(env); + const authSecretDir = cellAuthSecretDir(stateDir, record.tenantId); + const purgeTargets: string[] = []; + if (params.purgeData) { + const dataTarget = await resolvePurgeTarget( + path.join(stateDir, "fleet", "cells"), + record.dataDir, + record.tenantId, + ); + if (dataTarget) { + purgeTargets.push(dataTarget); + } + const authTarget = await resolvePurgeTarget( + path.join(stateDir, "fleet", "auth-profile-secrets"), + authSecretDir, + record.tenantId, + ); + if (authTarget) { + purgeTargets.push(authTarget); + } + } + const inspection = await containers.inspect(record.runtime, record.containerName); + if (inspection.kind === "unavailable") { + throw new Error( + `Cannot inspect ${record.runtime} container for tenant ${record.tenantId}: ${inspection.error}`, + ); + } + const networkName = cellNetworkName(record.tenantId); + const networkInspection = await containers.inspectNetwork(record.runtime, networkName); + if (networkInspection.kind === "unavailable") { + throw new Error( + `Cannot inspect ${record.runtime} network for tenant ${record.tenantId}: ${networkInspection.error}`, + ); + } + if (networkInspection.kind === "ok") { + assertManagedNetwork(record, networkInspection); + } + if (inspection.kind === "ok") { + assertManagedInspection(record, inspection); + if (inspection.running && !params.force) { + throw new Error( + `Fleet cell ${record.tenantId} is running; use --force to remove it.`, + ); + } + checkpoint(); + await containers.remove(record.runtime, record.containerName, params.force === true); + } + if (networkInspection.kind === "ok") { + checkpoint(); + await containers.removeNetwork(record.runtime, networkName); + } + if (purgeTargets.length > 0) { + checkpoint(); + await Promise.all( + purgeTargets.map((target) => fs.rm(target, { recursive: true, force: true })), + ); + } + checkpoint(); + deleteFleetCell(env, record.tenantId); + return { + tenant: record.tenantId, + action: "rm", + dataPurged: params.purgeData === true, + }; + }, + }); + }, + }; +} + +export type FleetService = ReturnType; diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 4da7c8f9bbc2..365349cfb30a 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -519,6 +519,16 @@ export interface ExecApprovalsConfig { updated_at_ms: number; } +export interface FleetCells { + container_name: string; + created_at_ms: number; + data_dir: string; + host_port: number; + image: string; + runtime: string; + tenant_id: string; +} + export interface FlowRuns { blocked_summary: string | null; blocked_task_id: string | null; @@ -1086,6 +1096,7 @@ export interface DB { diagnostic_events: DiagnosticEvents; diagnostic_stability_bundles: DiagnosticStabilityBundles; exec_approvals_config: ExecApprovalsConfig; + fleet_cells: FleetCells; flow_runs: FlowRuns; gateway_boot_lifecycle: GatewayBootLifecycle; gateway_restart_handoff: GatewayRestartHandoff; diff --git a/src/state/openclaw-state-schema.generated.ts b/src/state/openclaw-state-schema.generated.ts index c291a9be1ce2..fc212c3a0f06 100644 --- a/src/state/openclaw-state-schema.generated.ts +++ b/src/state/openclaw-state-schema.generated.ts @@ -1379,4 +1379,14 @@ CREATE TABLE IF NOT EXISTS worker_environments ( CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_environments_provider_lease ON worker_environments(provider_id, lease_id) - WHERE lease_id IS NOT NULL;\n`; + WHERE lease_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS fleet_cells ( + tenant_id TEXT NOT NULL PRIMARY KEY, + created_at_ms INTEGER NOT NULL, + image TEXT NOT NULL, + runtime TEXT NOT NULL, + host_port INTEGER NOT NULL, + container_name TEXT NOT NULL, + data_dir TEXT NOT NULL +);\n`; diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index 6f1d50f780cf..6a2ab256197a 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -1375,3 +1375,13 @@ CREATE TABLE IF NOT EXISTS worker_environments ( CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_environments_provider_lease ON worker_environments(provider_id, lease_id) WHERE lease_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS fleet_cells ( + tenant_id TEXT NOT NULL PRIMARY KEY, + created_at_ms INTEGER NOT NULL, + image TEXT NOT NULL, + runtime TEXT NOT NULL, + host_port INTEGER NOT NULL, + container_name TEXT NOT NULL, + data_dir TEXT NOT NULL +);