diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 9577d3e0063..5a1e6e7415c 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -430,6 +430,11 @@ jobs: command: pnpm test:docker:doctor-switch timeout_minutes: 60 release_path: true + - suite_id: docker-session-runtime-context + label: Session Runtime Context Docker E2E + command: pnpm test:docker:session-runtime-context + timeout_minutes: 60 + release_path: true - suite_id: docker-qr label: QR Import Docker E2E command: pnpm test:docker:qr diff --git a/CHANGELOG.md b/CHANGELOG.md index 30068bc5419..e2d4f991ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,10 @@ Docs: https://docs.openclaw.ai metadata when callers request the model without the provider prefix, so custom image models keep their `input: ["text", "image"]` capability. Fixes #33185. Thanks @Kobe9312 and @vincentkoc. +- Sessions: keep embedded runtime context out of the visible user prompt by + sending it as a hidden next-turn custom message, and teach doctor to repair + affected 2026.4.24 transcripts with duplicated prompt-rewrite branches. + Fixes #71761. - Gateway/subagents: keep direct-loopback backend RPCs authenticated with the shared gateway token/password off stale CLI paired-device scope baselines, so internal calls no longer hit `scope-upgrade` pairing prompts while remote, diff --git a/docs/gateway/doctor.md b/docs/gateway/doctor.md index 8813368d06e..a120c319f14 100644 --- a/docs/gateway/doctor.md +++ b/docs/gateway/doctor.md @@ -70,6 +70,7 @@ cat ~/.openclaw/openclaw.json - Legacy plugin manifest contract key migration (`speechProviders`, `realtimeTranscriptionProviders`, `realtimeVoiceProviders`, `mediaUnderstandingProviders`, `imageGenerationProviders`, `videoGenerationProviders`, `webFetchProviders`, `webSearchProviders` → `contracts`). - Legacy cron store migration (`jobId`, `schedule.cron`, top-level delivery/payload fields, payload `provider`, simple `notify: true` webhook fallback jobs). - Session lock file inspection and stale lock cleanup. +- Session transcript repair for duplicated prompt-rewrite branches created by affected 2026.4.24 builds. - State integrity and permissions checks (sessions, transcripts, state dir). - Config file permission checks (chmod 600) when running locally. - Model auth health: checks OAuth expiry, can refresh expiring tokens, and reports auth-profile cooldown/disabled states. @@ -317,6 +318,15 @@ considered stale (dead PID or older than 30 minutes). In `--fix` / `--repair` mode it removes stale lock files automatically; otherwise it prints a note and instructs you to rerun with `--fix`. +### 3d) Session transcript branch repair + +Doctor scans agent session JSONL files for the duplicated branch shape created +by the 2026.4.24 prompt transcript rewrite bug: an abandoned user turn with +OpenClaw internal runtime context plus an active sibling containing the same +visible user prompt. In `--fix` / `--repair` mode, doctor backs up each affected +file next to the original and rewrites the transcript to the active branch so +gateway history and memory readers no longer see duplicate turns. + ### 4) State integrity checks (session persistence, routing, and safety) The state directory is the operational brainstem. If it vanishes, you lose diff --git a/docs/help/testing.md b/docs/help/testing.md index ccf076c019f..a858bde2de7 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -120,6 +120,12 @@ runs the same lanes before release approval. endpoint. - Use `OPENCLAW_NPM_ONBOARD_CHANNEL=discord` to run the same packaged-install lane with Discord. +- `pnpm test:docker:session-runtime-context` + - Runs a deterministic built-app Docker smoke for embedded runtime context + transcripts. It verifies hidden OpenClaw runtime context is persisted as a + non-display custom message instead of leaking into the visible user turn, + then seeds an affected broken session JSONL and verifies + `openclaw doctor --fix` rewrites it to the active branch with a backup. - `pnpm test:docker:npm-telegram-live` - Installs a published OpenClaw package in Docker, runs installed-package onboarding, configures Telegram through the installed CLI, then reuses the @@ -587,7 +593,7 @@ These Docker runners split into two buckets: `OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS=90000`. Override those env vars when you explicitly want the larger exhaustive scan. - `test:docker:all` builds the live Docker image once via `test:docker:live-build`, then reuses it for the live Docker lanes. It also builds one shared `scripts/e2e/Dockerfile` image via `test:docker:e2e-build` and reuses it for the E2E container smoke runners that exercise the built app. The aggregate uses a weighted local scheduler: `OPENCLAW_DOCKER_ALL_PARALLELISM` controls process slots, while resource caps keep heavy live, npm-install, and multi-service lanes from all starting at once. Defaults are 10 slots, `OPENCLAW_DOCKER_ALL_LIVE_LIMIT=6`, `OPENCLAW_DOCKER_ALL_NPM_LIMIT=8`, and `OPENCLAW_DOCKER_ALL_SERVICE_LIMIT=7`; tune `OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT` or `OPENCLAW_DOCKER_ALL_DOCKER_LIMIT` only when the Docker host has more headroom. The runner performs a Docker preflight by default, removes stale OpenClaw E2E containers, prints status every 30 seconds, stores successful lane timings in `.artifacts/docker-tests/lane-timings.json`, and uses those timings to start longer lanes first on later runs. Use `OPENCLAW_DOCKER_ALL_DRY_RUN=1` to print the weighted lane manifest without building or running Docker. -- Container smoke runners: `test:docker:openwebui`, `test:docker:onboard`, `test:docker:npm-onboard-channel-agent`, `test:docker:agents-delete-shared-workspace`, `test:docker:gateway-network`, `test:docker:mcp-channels`, `test:docker:pi-bundle-mcp-tools`, `test:docker:cron-mcp-cleanup`, `test:docker:plugins`, `test:docker:plugin-update`, and `test:docker:config-reload` boot one or more real containers and verify higher-level integration paths. +- Container smoke runners: `test:docker:openwebui`, `test:docker:onboard`, `test:docker:npm-onboard-channel-agent`, `test:docker:session-runtime-context`, `test:docker:agents-delete-shared-workspace`, `test:docker:gateway-network`, `test:docker:mcp-channels`, `test:docker:pi-bundle-mcp-tools`, `test:docker:cron-mcp-cleanup`, `test:docker:plugins`, `test:docker:plugin-update`, and `test:docker:config-reload` boot one or more real containers and verify higher-level integration paths. The live-model Docker runners also bind-mount only the needed CLI auth homes (or all supported ones when the run is not narrowed), then copy them into the container home before the run so external-CLI OAuth can refresh tokens without mutating the host auth store: @@ -599,6 +605,7 @@ The live-model Docker runners also bind-mount only the needed CLI auth homes (or - Open WebUI live smoke: `pnpm test:docker:openwebui` (script: `scripts/e2e/openwebui-docker.sh`) - Onboarding wizard (TTY, full scaffolding): `pnpm test:docker:onboard` (script: `scripts/e2e/onboard-docker.sh`) - Npm tarball onboarding/channel/agent smoke: `pnpm test:docker:npm-onboard-channel-agent` installs the packed OpenClaw tarball globally in Docker, configures OpenAI via env-ref onboarding plus Telegram by default, verifies doctor repairs activated plugin runtime deps, and runs one mocked OpenAI agent turn. Reuse a prebuilt tarball with `OPENCLAW_NPM_ONBOARD_PACKAGE_TGZ=/path/to/openclaw-*.tgz`, skip the host rebuild with `OPENCLAW_NPM_ONBOARD_HOST_BUILD=0`, or switch channel with `OPENCLAW_NPM_ONBOARD_CHANNEL=discord`. +- Session runtime context smoke: `pnpm test:docker:session-runtime-context` verifies hidden runtime context transcript persistence plus doctor repair of affected duplicated prompt-rewrite branches. - Bun global install smoke: `bash scripts/e2e/bun-global-install-smoke.sh` packs the current tree, installs it with `bun install -g` in an isolated home, and verifies `openclaw infer image providers --json` returns bundled image providers instead of hanging. Reuse a prebuilt tarball with `OPENCLAW_BUN_GLOBAL_SMOKE_PACKAGE_TGZ=/path/to/openclaw-*.tgz`, skip the host build with `OPENCLAW_BUN_GLOBAL_SMOKE_HOST_BUILD=0`, or copy `dist/` from a built Docker image with `OPENCLAW_BUN_GLOBAL_SMOKE_DIST_IMAGE=openclaw-dockerfile-smoke:local`. - Installer Docker smoke: `bash scripts/test-install-sh-docker.sh` shares one npm cache across its root, update, and direct-npm containers. Update smoke defaults to npm `latest` as the stable baseline before upgrading to the candidate tarball. Non-root installer checks keep an isolated npm cache so root-owned cache entries do not mask user-local install behavior. Set `OPENCLAW_INSTALL_SMOKE_NPM_CACHE_DIR=/path/to/cache` to reuse the root/update/direct-npm cache across local reruns. - Install Smoke CI skips the duplicate direct-npm global update with `OPENCLAW_INSTALL_SMOKE_SKIP_NPM_GLOBAL=1`; run the script locally without that env when direct `npm install -g` coverage is needed. diff --git a/package.json b/package.json index ceae8fd7d9b..4edd30d858c 100644 --- a/package.json +++ b/package.json @@ -1531,6 +1531,7 @@ "test:docker:plugin-update": "bash scripts/e2e/plugin-update-unchanged-docker.sh", "test:docker:plugins": "bash scripts/e2e/plugins-docker.sh", "test:docker:qr": "bash scripts/e2e/qr-import-docker.sh", + "test:docker:session-runtime-context": "bash scripts/e2e/session-runtime-context-docker.sh", "test:e2e": "node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts", "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts extensions/openshell/src/backend.e2e.test.ts", "test:extension": "node scripts/test-extension.mjs", diff --git a/scripts/e2e/session-runtime-context-docker-client.ts b/scripts/e2e/session-runtime-context-docker-client.ts new file mode 100644 index 00000000000..a8cd145363e --- /dev/null +++ b/scripts/e2e/session-runtime-context-docker-client.ts @@ -0,0 +1,260 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { SessionManager } from "@mariozechner/pi-coding-agent"; +import { + queueRuntimeContextForNextTurn, + resolveRuntimeContextPromptParts, +} from "../../src/agents/pi-embedded-runner/run/runtime-context-prompt.js"; + +type TranscriptEntry = { + type?: string; + customType?: string; + content?: string; + display?: boolean; + message?: { + role?: string; + content?: unknown; + }; +}; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +async function readJsonl(filePath: string): Promise { + const raw = await fs.readFile(filePath, "utf-8"); + return raw + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line) as TranscriptEntry); +} + +function messageText(content: unknown): string { + if (typeof content === "string") { + return content; + } + if (!Array.isArray(content)) { + return ""; + } + return content + .map((part) => + part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string" + ? (part as { text: string }).text + : "", + ) + .join(""); +} + +async function verifyRuntimeContextTranscriptShape(root: string) { + const sessionFile = path.join(root, ".openclaw", "agents", "main", "sessions", "runtime.jsonl"); + await fs.mkdir(path.dirname(sessionFile), { recursive: true }); + const sessionManager = SessionManager.open(sessionFile); + const effectivePrompt = [ + "visible ask", + "", + "<<>>", + "secret docker context", + "<<>>", + ].join("\n"); + const promptSubmission = resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: "visible ask", + }); + + assert(promptSubmission.prompt === "visible ask", "visible prompt was not preserved"); + assert( + promptSubmission.runtimeContext?.includes("secret docker context"), + "runtime context was not extracted", + ); + + await queueRuntimeContextForNextTurn({ + runtimeContext: promptSubmission.runtimeContext, + session: { + sendCustomMessage: async (message, options) => { + assert(options?.deliverAs === "nextTurn", "runtime context was not queued for next turn"); + sessionManager.appendCustomMessageEntry( + message.customType, + message.content, + message.display, + message.details, + ); + }, + }, + }); + sessionManager.appendMessage({ + role: "user", + content: promptSubmission.prompt, + timestamp: Date.now(), + }); + sessionManager.appendMessage({ + role: "assistant", + content: "done", + timestamp: Date.now() + 1, + }); + + const entries = await readJsonl(sessionFile); + const customEntry = entries.find((entry) => entry.type === "custom_message"); + assert(customEntry, "hidden runtime custom message was not persisted"); + assert(customEntry.customType === "openclaw.runtime-context", "unexpected custom message type"); + assert(customEntry.display === false, "runtime custom message should be hidden"); + assert( + customEntry.content?.includes("secret docker context"), + "runtime custom message lost context", + ); + + const userEntries = entries.filter((entry) => entry.message?.role === "user"); + assert(userEntries.length === 1, `expected one visible user message, got ${userEntries.length}`); + const userText = messageText(userEntries[0]?.message?.content); + assert(userText === "visible ask", `unexpected visible user text: ${JSON.stringify(userText)}`); + assert( + !userText.includes("OPENCLAW_INTERNAL_CONTEXT") && !userText.includes("secret docker context"), + "visible user transcript leaked runtime context", + ); +} + +async function seedBrokenSession(stateDir: string): Promise { + const sessionsDir = path.join(stateDir, "agents", "main", "sessions"); + const sessionFile = path.join(sessionsDir, "broken.jsonl"); + await fs.mkdir(sessionsDir, { recursive: true }); + const entries = [ + { type: "session", version: 3, id: "broken-session" }, + { + type: "message", + id: "parent", + parentId: null, + message: { role: "assistant", content: "previous" }, + }, + { + type: "message", + id: "runtime-user", + parentId: "parent", + message: { + role: "user", + content: [ + "visible ask", + "", + "<<>>", + "secret doctor context", + "<<>>", + ].join("\n"), + }, + }, + { + type: "message", + id: "runtime-assistant", + parentId: "runtime-user", + message: { role: "assistant", content: "stale branch" }, + }, + { + type: "message", + id: "plain-user", + parentId: "parent", + message: { role: "user", content: "visible ask" }, + }, + { + type: "message", + id: "plain-assistant", + parentId: "plain-user", + message: { role: "assistant", content: "active answer" }, + }, + ]; + await fs.writeFile( + sessionFile, + `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`, + "utf-8", + ); + await fs.writeFile( + path.join(sessionsDir, "sessions.json"), + JSON.stringify( + { + "agent:main:qa:docker-runtime-context": { + sessionId: "broken", + sessionFile: "broken.jsonl", + updatedAt: Date.now(), + displayName: "Docker runtime context repair", + }, + }, + null, + 2, + ), + "utf-8", + ); + return sessionFile; +} + +async function verifyDoctorRepair(root: string) { + const stateDir = path.join(root, ".openclaw"); + const configPath = path.join(stateDir, "openclaw.json"); + const sessionFile = await seedBrokenSession(stateDir); + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile(configPath, JSON.stringify({ plugins: { enabled: false } }, null, 2)); + + const entry = await fs.stat("dist/index.mjs").then( + () => "dist/index.mjs", + () => "dist/index.js", + ); + const result = spawnSync(process.execPath, [entry, "doctor", "--fix", "--yes", "--force"], { + cwd: process.cwd(), + env: { + ...process.env, + HOME: root, + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_DISABLE_BONJOUR: "1", + OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1", + OPENCLAW_NO_ONBOARD: "1", + OPENCLAW_STATE_DIR: stateDir, + OPENCLAW_SKIP_CANVAS_HOST: "1", + OPENCLAW_SKIP_CHANNELS: "1", + OPENCLAW_SKIP_CRON: "1", + OPENCLAW_SKIP_GMAIL_WATCHER: "1", + }, + encoding: "utf-8", + timeout: 120_000, + }); + + assert( + result.status === 0, + `doctor --fix failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + const entries = await readJsonl(sessionFile); + const ids = entries.map((entry) => (entry as { id?: string }).id).filter(Boolean); + assert( + JSON.stringify(ids) === + JSON.stringify(["broken-session", "parent", "plain-user", "plain-assistant"]), + `doctor kept wrong active branch: ${JSON.stringify(ids)}`, + ); + assert( + entries.every( + (entry) => !messageText(entry.message?.content).includes("secret doctor context"), + ), + "doctor repair left runtime context in active transcript", + ); + const backups = (await fs.readdir(path.dirname(sessionFile))).filter((name) => + name.includes(".pre-doctor-branch-repair-"), + ); + assert(backups.length === 1, `expected one doctor backup, got ${backups.length}`); +} + +async function main() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-runtime-context-")); + process.env.HOME = root; + process.env.OPENCLAW_STATE_DIR = path.join(root, ".openclaw"); + process.env.OPENCLAW_CONFIG_PATH = path.join(process.env.OPENCLAW_STATE_DIR, "openclaw.json"); + try { + await verifyRuntimeContextTranscriptShape(root); + await verifyDoctorRepair(root); + console.log("session runtime context Docker E2E passed"); + } finally { + if (process.env.OPENCLAW_SESSION_RUNTIME_CONTEXT_KEEP_ARTIFACTS !== "1") { + await fs.rm(root, { recursive: true, force: true }); + } else { + console.error(`kept artifacts: ${root}`); + } + } +} + +await main(); diff --git a/scripts/e2e/session-runtime-context-docker.sh b/scripts/e2e/session-runtime-context-docker.sh new file mode 100644 index 00000000000..a057c14b175 --- /dev/null +++ b/scripts/e2e/session-runtime-context-docker.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh" + +IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-session-runtime-context-e2e" OPENCLAW_SESSION_RUNTIME_CONTEXT_E2E_IMAGE)" +CONTAINER_NAME="openclaw-session-runtime-context-e2e-$$" +RUN_LOG="$(mktemp -t openclaw-session-runtime-context-log.XXXXXX)" + +cleanup() { + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + rm -f "$RUN_LOG" +} +trap cleanup EXIT + +docker_e2e_build_or_reuse "$IMAGE_NAME" session-runtime-context + +echo "Running session runtime context Docker E2E..." +set +e +docker run --rm \ + --name "$CONTAINER_NAME" \ + -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + "$IMAGE_NAME" \ + bash -lc 'set -euo pipefail; node --import tsx scripts/e2e/session-runtime-context-docker-client.ts' \ + >"$RUN_LOG" 2>&1 +status=$? +set -e + +if [ "$status" -ne 0 ]; then + echo "Docker session runtime context smoke failed" + cat "$RUN_LOG" + exit "$status" +fi + +echo "OK" diff --git a/scripts/test-docker-all.mjs b/scripts/test-docker-all.mjs index a904073fcc3..40fe5dfcaff 100644 --- a/scripts/test-docker-all.mjs +++ b/scripts/test-docker-all.mjs @@ -254,6 +254,10 @@ const lanes = [ "crestodian-first-run", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:crestodian-first-run", ), + lane( + "session-runtime-context", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:session-runtime-context", + ), lane("qr", "pnpm test:docker:qr"), ]; diff --git a/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.context-engine.test.ts b/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.context-engine.test.ts index 4fbf273db79..434d76f4618 100644 --- a/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.context-engine.test.ts +++ b/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.context-engine.test.ts @@ -1,3 +1,5 @@ +import fs from "node:fs/promises"; +import path from "node:path"; import type { AgentMessage } from "@mariozechner/pi-agent-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildMemorySystemPromptAddition } from "../../../plugin-sdk/core.js"; @@ -18,6 +20,7 @@ import { import { cleanupTempPaths, createContextEngineBootstrapAndAssemble, + createContextEngineAttemptRunner, expectCalledWithSessionKey, getHoisted, resetEmbeddedAttemptHarness, @@ -33,6 +36,7 @@ const sessionFile = "/tmp/session.jsonl"; const seedMessage = { role: "user", content: "seed", timestamp: 1 } as AgentMessage; const doneMessage = { role: "assistant", content: "done", timestamp: 2 } as unknown as AgentMessage; type AfterTurnPromptCacheCall = { runtimeContext?: { promptCache?: Record } }; +type TrajectoryEvent = { type?: string; data?: Record }; function createTestContextEngine(params: Partial): AttemptContextEngine { return { @@ -125,6 +129,71 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => { vi.restoreAllMocks(); }); + it("sends transcriptPrompt visibly and queues runtime context as hidden custom context", async () => { + const seen: { prompt?: string; messages?: unknown[] } = {}; + + const result = await createContextEngineAttemptRunner({ + contextEngine: createContextEngineBootstrapAndAssemble(), + sessionKey, + tempPaths, + attemptOverrides: { + prompt: [ + "visible ask", + "", + "<<>>", + "secret runtime context", + "<<>>", + ].join("\n"), + transcriptPrompt: "visible ask", + }, + sessionPrompt: async (session, prompt) => { + seen.prompt = prompt; + seen.messages = [...session.messages]; + session.messages = [ + ...session.messages, + { role: "assistant", content: "done", timestamp: 2 }, + ]; + }, + }); + + expect(seen.prompt).toBe("visible ask"); + expect(result.finalPromptText).toBe("visible ask"); + expect(seen.messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + role: "custom", + customType: "openclaw.runtime-context", + display: false, + content: expect.stringContaining("secret runtime context"), + }), + ]), + ); + const trajectoryEvents = ( + await fs.readFile(path.join(tempPaths[0] ?? "", "session.trajectory.jsonl"), "utf8") + ) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as TrajectoryEvent); + const promptSubmitted = trajectoryEvents.find((event) => event.type === "prompt.submitted"); + const contextCompiled = trajectoryEvents.find((event) => event.type === "context.compiled"); + const modelCompleted = trajectoryEvents.find((event) => event.type === "model.completed"); + const traceArtifacts = trajectoryEvents.find((event) => event.type === "trace.artifacts"); + + expect(promptSubmitted?.data?.prompt).toBe("visible ask"); + expect(contextCompiled?.data?.prompt).toBe("visible ask"); + expect(modelCompleted?.data?.finalPromptText).toBe("visible ask"); + expect(traceArtifacts?.data?.finalPromptText).toBe("visible ask"); + for (const value of [ + promptSubmitted?.data?.prompt, + contextCompiled?.data?.prompt, + modelCompleted?.data?.finalPromptText, + traceArtifacts?.data?.finalPromptText, + ]) { + expect(String(value)).not.toContain("OPENCLAW_INTERNAL_CONTEXT"); + expect(String(value)).not.toContain("secret runtime context"); + } + }); + it("forwards sessionKey to bootstrap, assemble, and afterTurn", async () => { const { bootstrap, assemble } = createContextEngineBootstrapAndAssemble(); const afterTurn = vi.fn(async (_params: { sessionKey?: string }) => {}); diff --git a/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.test-support.ts b/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.test-support.ts index a6fccb82aed..2f56d8b97fb 100644 --- a/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.test-support.ts +++ b/src/agents/pi-embedded-runner/run/attempt.spawn-workspace.test-support.ts @@ -675,6 +675,15 @@ export type MutableSession = { }; }; prompt: (prompt: string, options?: { images?: unknown[] }) => Promise; + sendCustomMessage: ( + message: { + customType: string; + content: string; + display: boolean; + details?: Record; + }, + options?: { deliverAs?: "nextTurn"; triggerTurn?: boolean }, + ) => Promise; setActiveToolsByName: (toolNames: string[]) => void; abort: () => Promise; dispose: () => void; @@ -799,6 +808,11 @@ export function createDefaultEmbeddedSession(params?: { { role: "assistant", content: "done", timestamp: 2 }, ]; }, + sendCustomMessage: async (message, options) => { + if (options?.deliverAs === "nextTurn") { + session.messages = [...session.messages, { role: "custom", timestamp: 1, ...message }]; + } + }, abort: async () => {}, dispose: () => {}, steer: async () => {}, diff --git a/src/agents/pi-embedded-runner/run/attempt.ts b/src/agents/pi-embedded-runner/run/attempt.ts index b23e3bf93ee..6b8b1fee398 100644 --- a/src/agents/pi-embedded-runner/run/attempt.ts +++ b/src/agents/pi-embedded-runner/run/attempt.ts @@ -305,7 +305,10 @@ import { PREEMPTIVE_OVERFLOW_ERROR_TEXT, shouldPreemptivelyCompactBeforePrompt, } from "./preemptive-compaction.js"; -import { rewriteSubmittedPromptTranscript } from "./transcript-prompt-rewrite.js"; +import { + queueRuntimeContextForNextTurn, + resolveRuntimeContextPromptParts, +} from "./runtime-context-prompt.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js"; export { @@ -2373,10 +2376,15 @@ export async function runEmbeddedAttempt( } prePromptMessageCount = activeSession.messages.length; - // Detect and load images referenced in the prompt for vision-capable models. + const promptSubmission = resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: params.transcriptPrompt, + }); + + // Detect and load images referenced in the visible prompt for vision-capable models. // Images are prompt-local only (pi-like behavior). const imageResult = await detectAndLoadPromptImages({ - prompt: effectivePrompt, + prompt: promptSubmission.prompt, workspaceDir: effectiveWorkspace, model: params.model, existingImages: params.images, @@ -2392,13 +2400,13 @@ export async function runEmbeddedAttempt( }); cacheTrace?.recordStage("prompt:images", { - prompt: effectivePrompt, + prompt: promptSubmission.prompt, messages: activeSession.messages, note: `images: prompt=${imageResult.images.length}`, }); trajectoryRecorder?.recordEvent("context.compiled", { systemPrompt: systemPromptText, - prompt: effectivePrompt, + prompt: promptSubmission.prompt, messages: activeSession.messages, tools: toTrajectoryToolDefinitions(effectiveTools), imagesCount: imageResult.images.length, @@ -2410,7 +2418,7 @@ export async function runEmbeddedAttempt( if ( !skipPromptSubmission && !hasPromptSubmissionContent({ - prompt: effectivePrompt, + prompt: promptSubmission.prompt, messages: activeSession.messages, imageCount: imageResult.images.length, }) @@ -2423,7 +2431,7 @@ export async function runEmbeddedAttempt( ); trajectoryRecorder?.recordEvent("prompt.skipped", { reason: "empty_prompt_history_images", - prompt: effectivePrompt, + prompt: promptSubmission.prompt, messages: activeSession.messages, imagesCount: imageResult.images.length, }); @@ -2589,9 +2597,9 @@ export async function runEmbeddedAttempt( if (normalizedReplayMessages !== activeSession.messages) { activeSession.agent.state.messages = normalizedReplayMessages; } - finalPromptText = effectivePrompt; + finalPromptText = promptSubmission.prompt; trajectoryRecorder?.recordEvent("prompt.submitted", { - prompt: effectivePrompt, + prompt: promptSubmission.prompt, systemPrompt: systemPromptText, messages: activeSession.messages, imagesCount: imageResult.images.length, @@ -2600,25 +2608,22 @@ export async function runEmbeddedAttempt( updateActiveEmbeddedRunSnapshot(params.sessionId, { transcriptLeafId, messages: btwSnapshotMessages, - inFlightPrompt: effectivePrompt, + inFlightPrompt: promptSubmission.prompt, + }); + await queueRuntimeContextForNextTurn({ + session: activeSession, + runtimeContext: promptSubmission.runtimeContext, }); // Only pass images option if there are actually images to pass // This avoids potential issues with models that don't expect the images parameter if (imageResult.images.length > 0) { await abortable( - activeSession.prompt(effectivePrompt, { images: imageResult.images }), + activeSession.prompt(promptSubmission.prompt, { images: imageResult.images }), ); } else { - await abortable(activeSession.prompt(effectivePrompt)); + await abortable(activeSession.prompt(promptSubmission.prompt)); } - rewriteSubmittedPromptTranscript({ - sessionManager, - sessionFile: params.sessionFile, - previousLeafId: transcriptLeafId, - submittedPrompt: effectivePrompt, - transcriptPrompt: params.transcriptPrompt, - }); } } catch (err) { yieldAborted = diff --git a/src/agents/pi-embedded-runner/run/params.ts b/src/agents/pi-embedded-runner/run/params.ts index 4e51811ad50..2278766d0d9 100644 --- a/src/agents/pi-embedded-runner/run/params.ts +++ b/src/agents/pi-embedded-runner/run/params.ts @@ -79,7 +79,7 @@ export type RunEmbeddedPiAgentParams = { config?: OpenClawConfig; skillsSnapshot?: SkillSnapshot; prompt: string; - /** User-visible prompt body to persist instead of runtime-enriched prompt text. */ + /** User-visible prompt body to submit and persist; runtime context travels separately. */ transcriptPrompt?: string; images?: ImageContent[]; imageOrder?: PromptImageOrderEntry[]; diff --git a/src/agents/pi-embedded-runner/run/runtime-context-prompt.test.ts b/src/agents/pi-embedded-runner/run/runtime-context-prompt.test.ts new file mode 100644 index 00000000000..65b1cc20360 --- /dev/null +++ b/src/agents/pi-embedded-runner/run/runtime-context-prompt.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from "vitest"; +import { + queueRuntimeContextForNextTurn, + resolveRuntimeContextPromptParts, +} from "./runtime-context-prompt.js"; + +describe("runtime context prompt submission", () => { + it("keeps unchanged prompts as a normal user prompt", () => { + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: "visible ask", + transcriptPrompt: "visible ask", + }), + ).toEqual({ prompt: "visible ask" }); + }); + + it("moves hidden runtime context out of the visible prompt", () => { + const effectivePrompt = [ + "visible ask", + "", + "<<>>", + "secret runtime context", + "<<>>", + ].join("\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: "visible ask", + }), + ).toEqual({ + prompt: "visible ask", + runtimeContext: + "<<>>\nsecret runtime context\n<<>>", + }); + }); + + it("preserves prompt additions as hidden runtime context", () => { + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: ["runtime prefix", "", "visible ask", "", "retry instruction"].join("\n"), + transcriptPrompt: "visible ask", + }), + ).toEqual({ + prompt: "visible ask", + runtimeContext: "runtime prefix\n\nretry instruction", + }); + }); + + it("uses a marker prompt for runtime-only events", () => { + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: "internal event", + transcriptPrompt: "", + }), + ).toEqual({ + prompt: "[OpenClaw runtime event]", + runtimeContext: "internal event", + }); + }); + + it("queues runtime context as a hidden next-turn custom message", async () => { + const sendCustomMessage = vi.fn(async () => {}); + + await queueRuntimeContextForNextTurn({ + session: { sendCustomMessage }, + runtimeContext: "secret runtime context", + }); + + expect(sendCustomMessage).toHaveBeenCalledWith( + expect.objectContaining({ + customType: "openclaw.runtime-context", + content: expect.stringContaining("secret runtime context"), + display: false, + }), + { deliverAs: "nextTurn" }, + ); + }); +}); diff --git a/src/agents/pi-embedded-runner/run/runtime-context-prompt.ts b/src/agents/pi-embedded-runner/run/runtime-context-prompt.ts new file mode 100644 index 00000000000..f705b1e0b03 --- /dev/null +++ b/src/agents/pi-embedded-runner/run/runtime-context-prompt.ts @@ -0,0 +1,68 @@ +const OPENCLAW_RUNTIME_CONTEXT_CUSTOM_TYPE = "openclaw.runtime-context"; +const EMPTY_RUNTIME_EVENT_PROMPT = "[OpenClaw runtime event]"; + +type RuntimeContextSession = { + sendCustomMessage: ( + message: { + customType: string; + content: string; + display: boolean; + details?: Record; + }, + options?: { deliverAs?: "nextTurn"; triggerTurn?: boolean }, + ) => Promise; +}; + +function removeLastPromptOccurrence(text: string, prompt: string): string | null { + const index = text.lastIndexOf(prompt); + if (index === -1) { + return null; + } + const before = text.slice(0, index).trimEnd(); + const after = text.slice(index + prompt.length).trimStart(); + return [before, after] + .filter((part) => part.length > 0) + .join("\n\n") + .trim(); +} + +export function resolveRuntimeContextPromptParts(params: { + effectivePrompt: string; + transcriptPrompt?: string; +}): { prompt: string; runtimeContext?: string } { + const transcriptPrompt = params.transcriptPrompt; + if (transcriptPrompt === undefined || transcriptPrompt === params.effectivePrompt) { + return { prompt: params.effectivePrompt }; + } + + const prompt = transcriptPrompt.trim() || EMPTY_RUNTIME_EVENT_PROMPT; + const runtimeContext = + removeLastPromptOccurrence(params.effectivePrompt, transcriptPrompt)?.trim() || + params.effectivePrompt.trim(); + + return runtimeContext ? { prompt, runtimeContext } : { prompt }; +} + +export async function queueRuntimeContextForNextTurn(params: { + session: RuntimeContextSession; + runtimeContext?: string; +}): Promise { + const runtimeContext = params.runtimeContext?.trim(); + if (!runtimeContext) { + return; + } + await params.session.sendCustomMessage( + { + customType: OPENCLAW_RUNTIME_CONTEXT_CUSTOM_TYPE, + content: [ + "OpenClaw runtime context for the immediately preceding user message.", + "This context is runtime-generated, not user-authored. Keep internal details private.", + "", + runtimeContext, + ].join("\n"), + display: false, + details: { source: "openclaw-runtime-context" }, + }, + { deliverAs: "nextTurn" }, + ); +} diff --git a/src/agents/pi-embedded-runner/run/transcript-prompt-rewrite.test.ts b/src/agents/pi-embedded-runner/run/transcript-prompt-rewrite.test.ts deleted file mode 100644 index 0d04bd98706..00000000000 --- a/src/agents/pi-embedded-runner/run/transcript-prompt-rewrite.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { SessionManager } from "@mariozechner/pi-coding-agent"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { onSessionTranscriptUpdate } from "../../../sessions/transcript-events.js"; -import { rewriteSubmittedPromptTranscript } from "./transcript-prompt-rewrite.js"; - -type AppendMessage = Parameters[0]; - -let tmpDir: string | undefined; - -async function createTmpDir(): Promise { - tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "transcript-prompt-rewrite-")); - return tmpDir; -} - -afterEach(async () => { - if (tmpDir) { - await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); - tmpDir = undefined; - } -}); - -function getUserTextMessages(sessionManager: SessionManager): string[] { - const messages: string[] = []; - for (const entry of sessionManager.getBranch()) { - if (entry.type !== "message" || entry.message.role !== "user") { - continue; - } - const content = (entry.message as { content?: unknown }).content; - if (typeof content === "string") { - messages.push(content); - continue; - } - if (!Array.isArray(content)) { - messages.push(""); - continue; - } - messages.push( - content - .map((block) => - block && - typeof block === "object" && - typeof (block as { text?: unknown }).text === "string" - ? (block as { text: string }).text - : "", - ) - .join(""), - ); - } - return messages; -} - -describe("rewriteSubmittedPromptTranscript", () => { - it("rewrites only the submitted embedded Pi prompt in a real session file", async () => { - const sessionDir = await createTmpDir(); - const sessionManager = SessionManager.create(sessionDir, sessionDir); - const submittedPrompt = - "visible ask\n\n<<>>\nsecret runtime context\n<<>>"; - const transcriptPrompt = "visible ask"; - - sessionManager.appendMessage({ - role: "user", - content: submittedPrompt, - timestamp: 1, - }); - const previousLeafId = sessionManager.appendMessage({ - role: "assistant", - content: [{ type: "text", text: "old answer" }], - timestamp: 2, - } as AppendMessage); - sessionManager.appendMessage({ - role: "user", - content: submittedPrompt, - timestamp: 3, - }); - const sessionFile = sessionManager.getSessionFile(); - expect(sessionFile).toBeTruthy(); - - const listener = vi.fn(); - const cleanup = onSessionTranscriptUpdate(listener); - try { - rewriteSubmittedPromptTranscript({ - sessionManager, - sessionFile: sessionFile!, - previousLeafId, - submittedPrompt, - transcriptPrompt, - }); - } finally { - cleanup(); - } - - expect(listener).toHaveBeenCalledWith({ sessionFile }); - - const reopenedSession = SessionManager.open(sessionFile!); - expect(getUserTextMessages(reopenedSession)).toEqual([submittedPrompt, transcriptPrompt]); - }); -}); diff --git a/src/agents/pi-embedded-runner/run/transcript-prompt-rewrite.ts b/src/agents/pi-embedded-runner/run/transcript-prompt-rewrite.ts deleted file mode 100644 index 5dafc4ebb58..00000000000 --- a/src/agents/pi-embedded-runner/run/transcript-prompt-rewrite.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { AgentMessage } from "@mariozechner/pi-agent-core"; -import { SessionManager } from "@mariozechner/pi-coding-agent"; -import { emitSessionTranscriptUpdate } from "../../../sessions/transcript-events.js"; -import { rewriteTranscriptEntriesInSessionManager } from "../transcript-rewrite.js"; - -type SessionManagerLike = ReturnType; - -function extractPromptTextFromMessage(message: AgentMessage): string | undefined { - const content = (message as { content?: unknown }).content; - if (typeof content === "string") { - return content; - } - if (!Array.isArray(content)) { - return undefined; - } - const textBlocks = content - .map((block) => - block && typeof block === "object" && typeof (block as { text?: unknown }).text === "string" - ? (block as { text: string }).text - : undefined, - ) - .filter((text): text is string => typeof text === "string"); - return textBlocks.length > 0 ? textBlocks.join("") : undefined; -} - -function replacePromptTextInMessage(message: AgentMessage, text: string): AgentMessage { - const content = (message as { content?: unknown }).content; - const entry = message as unknown as Record; - if (typeof content === "string") { - return { ...entry, content: text } as AgentMessage; - } - if (!Array.isArray(content)) { - return { ...entry, content: text } as AgentMessage; - } - let replaced = false; - const nextContent: unknown[] = []; - for (const block of content) { - if ( - replaced || - !block || - typeof block !== "object" || - typeof (block as { text?: unknown }).text !== "string" - ) { - nextContent.push(block); - continue; - } - replaced = true; - nextContent.push({ ...(block as Record), text }); - } - return { - ...entry, - content: replaced ? nextContent : text, - } as AgentMessage; -} - -export function rewriteSubmittedPromptTranscript(params: { - sessionManager: SessionManagerLike; - sessionFile: string; - previousLeafId: string | null; - submittedPrompt: string; - transcriptPrompt?: string; -}): void { - const transcriptPrompt = params.transcriptPrompt; - if (transcriptPrompt === undefined || transcriptPrompt === params.submittedPrompt) { - return; - } - const replacementText = transcriptPrompt.trim() || "[OpenClaw runtime event]"; - const branch = params.sessionManager.getBranch(); - const startIndex = params.previousLeafId - ? Math.max(0, branch.findIndex((entry) => entry.id === params.previousLeafId) + 1) - : 0; - const target = branch.slice(startIndex).find((entry) => { - if (entry.type !== "message" || entry.message.role !== "user") { - return false; - } - const text = extractPromptTextFromMessage(entry.message as AgentMessage); - return text === params.submittedPrompt; - }); - if (!target || target.type !== "message") { - return; - } - const result = rewriteTranscriptEntriesInSessionManager({ - sessionManager: params.sessionManager, - replacements: [ - { - entryId: target.id, - message: replacePromptTextInMessage(target.message, replacementText), - }, - ], - }); - if (result.changed) { - emitSessionTranscriptUpdate(params.sessionFile); - } -} diff --git a/src/commands/doctor-session-transcripts.test.ts b/src/commands/doctor-session-transcripts.test.ts new file mode 100644 index 00000000000..d7eb42721eb --- /dev/null +++ b/src/commands/doctor-session-transcripts.test.ts @@ -0,0 +1,155 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const note = vi.hoisted(() => vi.fn()); + +vi.mock("../terminal/note.js", () => ({ + note, +})); + +import { + noteSessionTranscriptHealth, + repairBrokenSessionTranscriptFile, +} from "./doctor-session-transcripts.js"; + +describe("doctor session transcript repair", () => { + let root: string; + + beforeEach(async () => { + note.mockClear(); + root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-doctor-transcripts-")); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + async function writeTranscript(entries: unknown[]): Promise { + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + const filePath = path.join(sessionsDir, "session.jsonl"); + await fs.writeFile(filePath, `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); + return filePath; + } + + it("rewrites affected prompt-rewrite branches to the active branch", async () => { + const filePath = await writeTranscript([ + { type: "session", version: 3, id: "session-1", timestamp: "2026-04-25T00:00:00Z" }, + { + type: "message", + id: "parent", + parentId: null, + message: { role: "assistant", content: "previous" }, + }, + { + type: "message", + id: "runtime-user", + parentId: "parent", + message: { + role: "user", + content: [ + "visible ask", + "", + "<<>>", + "secret", + "<<>>", + ].join("\n"), + }, + }, + { + type: "message", + id: "runtime-assistant", + parentId: "runtime-user", + message: { role: "assistant", content: "stale" }, + }, + { + type: "message", + id: "plain-user", + parentId: "parent", + message: { role: "user", content: "visible ask" }, + }, + { + type: "message", + id: "plain-assistant", + parentId: "plain-user", + message: { role: "assistant", content: "answer" }, + }, + ]); + + const result = await repairBrokenSessionTranscriptFile({ filePath, shouldRepair: true }); + + expect(result).toMatchObject({ + broken: true, + repaired: true, + originalEntries: 6, + activeEntries: 3, + }); + expect(result.backupPath).toBeTruthy(); + await expect(fs.access(result.backupPath!)).resolves.toBeUndefined(); + const lines = (await fs.readFile(filePath, "utf-8")).trim().split(/\r?\n/); + expect(lines).toHaveLength(4); + expect( + lines + .map((line) => JSON.parse(line)) + .filter((entry) => entry.type !== "session") + .map((entry) => entry.id), + ).toEqual(["parent", "plain-user", "plain-assistant"]); + }); + + it("reports affected transcripts without rewriting outside repair mode", async () => { + const filePath = await writeTranscript([ + { type: "session", version: 3, id: "session-1", timestamp: "2026-04-25T00:00:00Z" }, + { + type: "message", + id: "runtime-user", + parentId: null, + message: { + role: "user", + content: + "visible ask\n\n<<>>\nsecret\n<<>>", + }, + }, + { + type: "message", + id: "plain-user", + parentId: null, + message: { role: "user", content: "visible ask" }, + }, + ]); + const sessionsDir = path.dirname(filePath); + + await noteSessionTranscriptHealth({ shouldRepair: false, sessionDirs: [sessionsDir] }); + + expect(note).toHaveBeenCalledTimes(1); + const [message, title] = note.mock.calls[0] as [string, string]; + expect(title).toBe("Session transcripts"); + expect(message).toContain("duplicated prompt-rewrite branches"); + expect(message).toContain('Run "openclaw doctor --fix"'); + expect((await fs.readFile(filePath, "utf-8")).split(/\r?\n/).filter(Boolean)).toHaveLength(3); + }); + + it("ignores ordinary branch history without internal runtime context", async () => { + const filePath = await writeTranscript([ + { type: "session", version: 3, id: "session-1", timestamp: "2026-04-25T00:00:00Z" }, + { + type: "message", + id: "branch-a", + parentId: null, + message: { role: "user", content: "draft A" }, + }, + { + type: "message", + id: "branch-b", + parentId: null, + message: { role: "user", content: "draft B" }, + }, + ]); + + const result = await repairBrokenSessionTranscriptFile({ filePath, shouldRepair: true }); + + expect(result.broken).toBe(false); + expect((await fs.readFile(filePath, "utf-8")).split(/\r?\n/).filter(Boolean)).toHaveLength(3); + }); +}); diff --git a/src/commands/doctor-session-transcripts.ts b/src/commands/doctor-session-transcripts.ts new file mode 100644 index 00000000000..2fb10f64019 --- /dev/null +++ b/src/commands/doctor-session-transcripts.ts @@ -0,0 +1,295 @@ +import type { Dirent } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { + hasInternalRuntimeContext, + stripInternalRuntimeContext, +} from "../agents/internal-runtime-context.js"; +import { resolveAgentSessionDirs } from "../agents/session-dirs.js"; +import { resolveStateDir } from "../config/paths.js"; +import { note } from "../terminal/note.js"; +import { shortenHomePath } from "../utils.js"; + +type TranscriptEntry = Record & { + id?: unknown; + parentId?: unknown; + type?: unknown; + message?: unknown; +}; + +type TranscriptRepairResult = { + filePath: string; + broken: boolean; + repaired: boolean; + originalEntries: number; + activeEntries: number; + backupPath?: string; + reason?: string; +}; + +function parseTranscriptEntries(raw: string): TranscriptEntry[] { + const entries: TranscriptEntry[] = []; + for (const line of raw.split(/\r?\n/)) { + if (!line.trim()) { + continue; + } + try { + const parsed = JSON.parse(line); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + entries.push(parsed as TranscriptEntry); + } + } catch { + return []; + } + } + return entries; +} + +function getEntryId(entry: TranscriptEntry): string | null { + return typeof entry.id === "string" && entry.id.trim() ? entry.id : null; +} + +function getParentId(entry: TranscriptEntry): string | null { + return typeof entry.parentId === "string" && entry.parentId.trim() ? entry.parentId : null; +} + +function getMessage(entry: TranscriptEntry): Record | null { + return entry.message && typeof entry.message === "object" && !Array.isArray(entry.message) + ? (entry.message as Record) + : null; +} + +function textFromContent(content: unknown): string | null { + if (typeof content === "string") { + return content; + } + if (!Array.isArray(content)) { + return null; + } + const text = content + .map((part) => + part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string" + ? (part as { text: string }).text + : "", + ) + .join(""); + return text || null; +} + +function selectActivePath(entries: TranscriptEntry[]): TranscriptEntry[] | null { + const sessionEntries = entries.filter((entry) => entry.type !== "session"); + const leaf = sessionEntries.at(-1); + const leafId = leaf ? getEntryId(leaf) : null; + if (!leaf || !leafId) { + return null; + } + + const byId = new Map(); + for (const entry of sessionEntries) { + const id = getEntryId(entry); + if (id) { + byId.set(id, entry); + } + } + + const active: TranscriptEntry[] = []; + const seen = new Set(); + let current: TranscriptEntry | undefined = leaf; + while (current) { + const id = getEntryId(current); + if (!id || seen.has(id)) { + return null; + } + seen.add(id); + active.unshift(current); + const parentId = getParentId(current); + current = parentId ? byId.get(parentId) : undefined; + } + return active; +} + +function hasBrokenPromptRewriteBranch(entries: TranscriptEntry[], activePath: TranscriptEntry[]) { + const activeIds = new Set(activePath.map(getEntryId).filter((id): id is string => Boolean(id))); + const activeUserByParentAndText = new Set(); + + for (const entry of activePath) { + const id = getEntryId(entry); + const message = getMessage(entry); + if (!id || message?.role !== "user") { + continue; + } + const text = textFromContent(message.content); + if (text !== null) { + activeUserByParentAndText.add(`${getParentId(entry) ?? ""}\0${text.trim()}`); + } + } + + for (const entry of entries) { + const id = getEntryId(entry); + if (!id || activeIds.has(id)) { + continue; + } + const message = getMessage(entry); + if (message?.role !== "user") { + continue; + } + const text = textFromContent(message.content); + if (!text || !hasInternalRuntimeContext(text)) { + continue; + } + const visibleText = stripInternalRuntimeContext(text).trim(); + if ( + visibleText && + activeUserByParentAndText.has(`${getParentId(entry) ?? ""}\0${visibleText}`) + ) { + return true; + } + } + return false; +} + +async function writeActiveTranscript(params: { + filePath: string; + entries: TranscriptEntry[]; + activePath: TranscriptEntry[]; +}): Promise { + const header = params.entries.find((entry) => entry.type === "session"); + if (!header) { + throw new Error("missing session header"); + } + const backupPath = `${params.filePath}.pre-doctor-branch-repair-${new Date() + .toISOString() + .replace(/[:.]/g, "-")}.bak`; + await fs.copyFile(params.filePath, backupPath); + const next = [header, ...params.activePath].map((entry) => JSON.stringify(entry)).join("\n"); + await fs.writeFile(params.filePath, `${next}\n`, "utf-8"); + return backupPath; +} + +export async function repairBrokenSessionTranscriptFile(params: { + filePath: string; + shouldRepair: boolean; +}): Promise { + try { + const raw = await fs.readFile(params.filePath, "utf-8"); + const entries = parseTranscriptEntries(raw); + const activePath = selectActivePath(entries); + if (!activePath) { + return { + filePath: params.filePath, + broken: false, + repaired: false, + originalEntries: entries.length, + activeEntries: 0, + reason: "no active branch", + }; + } + const broken = hasBrokenPromptRewriteBranch(entries, activePath); + if (!broken) { + return { + filePath: params.filePath, + broken: false, + repaired: false, + originalEntries: entries.length, + activeEntries: activePath.length, + }; + } + if (!params.shouldRepair) { + return { + filePath: params.filePath, + broken: true, + repaired: false, + originalEntries: entries.length, + activeEntries: activePath.length, + }; + } + const backupPath = await writeActiveTranscript({ + filePath: params.filePath, + entries, + activePath, + }); + return { + filePath: params.filePath, + broken: true, + repaired: true, + originalEntries: entries.length, + activeEntries: activePath.length, + backupPath, + }; + } catch (err) { + return { + filePath: params.filePath, + broken: false, + repaired: false, + originalEntries: 0, + activeEntries: 0, + reason: String(err), + }; + } +} + +async function listSessionTranscriptFiles(sessionDirs: string[]): Promise { + const files: string[] = []; + for (const sessionsDir of sessionDirs) { + let entries: Dirent[] = []; + try { + entries = await fs.readdir(sessionsDir, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith(".jsonl")) { + files.push(path.join(sessionsDir, entry.name)); + } + } + } + return files.toSorted((a, b) => a.localeCompare(b)); +} + +export async function noteSessionTranscriptHealth(params?: { + shouldRepair?: boolean; + sessionDirs?: string[]; +}) { + const shouldRepair = params?.shouldRepair === true; + let sessionDirs = params?.sessionDirs; + try { + sessionDirs ??= await resolveAgentSessionDirs(resolveStateDir(process.env)); + } catch (err) { + note(`- Failed to inspect session transcripts: ${String(err)}`, "Session transcripts"); + return; + } + + const files = await listSessionTranscriptFiles(sessionDirs); + if (files.length === 0) { + return; + } + + const results: TranscriptRepairResult[] = []; + for (const filePath of files) { + results.push(await repairBrokenSessionTranscriptFile({ filePath, shouldRepair })); + } + const broken = results.filter((result) => result.broken); + if (broken.length === 0) { + return; + } + + const repairedCount = broken.filter((result) => result.repaired).length; + const lines = [ + `- Found ${broken.length} transcript file${broken.length === 1 ? "" : "s"} with duplicated prompt-rewrite branches.`, + ...broken.slice(0, 20).map((result) => { + const backup = result.backupPath ? ` backup=${shortenHomePath(result.backupPath)}` : ""; + const status = result.repaired ? "repaired" : "needs repair"; + return `- ${shortenHomePath(result.filePath)} ${status} entries=${result.originalEntries}->${result.activeEntries + 1}${backup}`; + }), + ]; + if (broken.length > 20) { + lines.push(`- ...and ${broken.length - 20} more.`); + } + if (!shouldRepair) { + lines.push('- Run "openclaw doctor --fix" to rewrite affected files to their active branch.'); + } else if (repairedCount > 0) { + lines.push(`- Repaired ${repairedCount} transcript file${repairedCount === 1 ? "" : "s"}.`); + } + + note(lines.join("\n"), "Session transcripts"); +} diff --git a/src/commands/doctor-state-integrity.test.ts b/src/commands/doctor-state-integrity.test.ts index d125b01b620..c01ef106d5d 100644 --- a/src/commands/doctor-state-integrity.test.ts +++ b/src/commands/doctor-state-integrity.test.ts @@ -302,6 +302,50 @@ describe("doctor state integrity oauth dir checks", () => { expect(files.some((name) => name.startsWith("orphan-session.jsonl.deleted."))).toBe(true); }); + it.skipIf(process.platform === "win32")( + "does not archive referenced transcripts when the state dir path resolves through a symlink", + async () => { + const cfg: OpenClawConfig = {}; + const originalHome = tempHome; + const symlinkHome = path.join( + path.dirname(originalHome), + `${path.basename(originalHome)}-link`, + ); + fs.symlinkSync(originalHome, symlinkHome, "dir"); + try { + process.env.HOME = symlinkHome; + process.env.OPENCLAW_HOME = symlinkHome; + process.env.OPENCLAW_STATE_DIR = path.join(symlinkHome, ".openclaw"); + + setupSessionState(cfg, process.env, symlinkHome); + const sessionsDir = resolveSessionTranscriptsDirForAgent( + "main", + process.env, + () => symlinkHome, + ); + const transcriptPath = path.join(sessionsDir, "linked-session.jsonl"); + fs.writeFileSync(transcriptPath, '{"type":"session"}\n'); + writeSessionStore(cfg, { + "agent:main:main": { + sessionId: "linked-session", + updatedAt: Date.now(), + }, + }); + + const confirmRuntimeRepair = vi.fn(async (params: { message: string }) => + params.message.includes("This only renames them to *.deleted.."), + ); + await noteStateIntegrity(cfg, { confirmRuntimeRepair, note: noteMock }); + + expect(fs.existsSync(transcriptPath)).toBe(true); + expect(fs.readdirSync(sessionsDir).some((name) => name.includes(".deleted."))).toBe(false); + expect(stateIntegrityText()).not.toContain("These .jsonl files are no longer referenced"); + } finally { + fs.rmSync(symlinkHome, { force: true }); + } + }, + ); + it("suppresses orphan transcript warnings when QMD sessions are enabled", async () => { const confirmRuntimeRepair = await runOrphanTranscriptCheckWithQmdSessions(true, tempHome); diff --git a/src/commands/doctor-state-integrity.ts b/src/commands/doctor-state-integrity.ts index 3b7ed456e6b..cd45d559b0c 100644 --- a/src/commands/doctor-state-integrity.ts +++ b/src/commands/doctor-state-integrity.ts @@ -74,6 +74,10 @@ function tryResolveNativeRealPath(targetPath: string): string | null { } } +function resolveComparableTranscriptPath(filePath: string): string { + return tryResolveNativeRealPath(filePath) ?? path.resolve(filePath); +} + function isReachableConfiguredAgentDir(params: { agentsRoot: string; dirName: string; @@ -888,7 +892,9 @@ export async function noteStateIntegrity( } try { referencedTranscriptPaths.add( - path.resolve(resolveSessionFilePath(entry.sessionId, entry, sessionPathOpts)), + resolveComparableTranscriptPath( + resolveSessionFilePath(entry.sessionId, entry, sessionPathOpts), + ), ); } catch { // ignore invalid legacy paths @@ -897,8 +903,10 @@ export async function noteStateIntegrity( const sessionDirEntries = fs.readdirSync(sessionsDir, { withFileTypes: true }); const orphanTranscriptPaths = sessionDirEntries .filter((entry) => entry.isFile() && isPrimarySessionTranscriptFileName(entry.name)) - .map((entry) => path.resolve(path.join(sessionsDir, entry.name))) - .filter((filePath) => !referencedTranscriptPaths.has(filePath)); + .map((entry) => path.join(sessionsDir, entry.name)) + .filter( + (filePath) => !referencedTranscriptPaths.has(resolveComparableTranscriptPath(filePath)), + ); if (orphanTranscriptPaths.length > 0 && !suppressOrphanTranscriptWarning) { const orphanCount = countLabel(orphanTranscriptPaths.length, "orphan transcript file"); const orphanPreview = formatFilePreview(orphanTranscriptPaths); diff --git a/src/commands/doctor.fast-path-mocks.ts b/src/commands/doctor.fast-path-mocks.ts index 10e9cf7d5ad..a0421fbb312 100644 --- a/src/commands/doctor.fast-path-mocks.ts +++ b/src/commands/doctor.fast-path-mocks.ts @@ -50,6 +50,10 @@ vi.mock("./doctor-session-locks.js", () => ({ noteSessionLockHealth: vi.fn().mockResolvedValue(undefined), })); +vi.mock("./doctor-session-transcripts.js", () => ({ + noteSessionTranscriptHealth: vi.fn().mockResolvedValue(undefined), +})); + vi.mock("./doctor-state-integrity.js", () => ({ noteStateIntegrity: vi.fn().mockResolvedValue(undefined), noteWorkspaceBackupTip: vi.fn(), diff --git a/src/flows/doctor-health-contributions.ts b/src/flows/doctor-health-contributions.ts index 570725303d6..d0dcee02b1d 100644 --- a/src/flows/doctor-health-contributions.ts +++ b/src/flows/doctor-health-contributions.ts @@ -253,6 +253,11 @@ async function runSessionLocksHealth(ctx: DoctorHealthFlowContext): Promise { + const { noteSessionTranscriptHealth } = await import("../commands/doctor-session-transcripts.js"); + await noteSessionTranscriptHealth({ shouldRepair: ctx.prompter.shouldRepair }); +} + async function runLegacyCronHealth(ctx: DoctorHealthFlowContext): Promise { const { maybeRepairLegacyCronStore } = await import("../commands/doctor-cron.js"); await maybeRepairLegacyCronStore({ @@ -572,6 +577,11 @@ export function resolveDoctorHealthContributions(): DoctorHealthContribution[] { label: "Session locks", run: runSessionLocksHealth, }), + createDoctorHealthContribution({ + id: "doctor:session-transcripts", + label: "Session transcripts", + run: runSessionTranscriptsHealth, + }), createDoctorHealthContribution({ id: "doctor:legacy-cron", label: "Legacy cron",