fix: promote serialized tool calls via repair package

Extracts serialized plaintext tool-call parsing, scrubbing, stream normalization, and standalone promotion into the private internal @openclaw/tool-call-repair package.

Provider wrappers and the embedded runner now share one repair path for standalone serialized tool calls, including adjacent text-block splits, while preserving exact argument bytes when already valid. The public plugin SDK payload module remains as the compatibility facade.

Verification:
- pnpm test src/plugin-sdk/provider-stream-shared.test.ts src/plugin-sdk/tool-payload.test.ts src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts -- --reporter=verbose
- env -u OPENCLAW_TESTBOX pnpm check:changed
- PR CI: all reported checks green/skipped/neutral on ff0b3c0a5c

Refs #86924

Co-authored-by: fuller-stack-dev <263060202+fuller-stack-dev@users.noreply.github.com>
This commit is contained in:
Jason (Json)
2026-05-30 15:34:57 -06:00
committed by GitHub
parent 443255461c
commit 3ea911558c
20 changed files with 5463 additions and 648 deletions

View File

@@ -0,0 +1,9 @@
{
"name": "@openclaw/tool-call-repair",
"version": "0.0.0-private",
"private": true,
"type": "module",
"exports": {
".": "./src/index.ts"
}
}

View File

@@ -0,0 +1,218 @@
export const END_TOOL_REQUEST = "[END_TOOL_REQUEST]";
export const HARMONY_CHANNEL_MARKER = "<|channel|>";
export const HARMONY_MESSAGE_MARKER = "<|message|>";
export const HARMONY_CALL_MARKER = "<|call|>";
export function matchesLiteralPrefix(text: string, literal: string): boolean {
return literal.startsWith(text) || text.startsWith(literal);
}
export function isPlainTextToolNameChar(char: string | undefined): boolean {
return Boolean(char && /[A-Za-z0-9_-]/.test(char));
}
export function isXmlishNameChar(char: string | undefined): boolean {
return Boolean(char && /[A-Za-z0-9_.:-]/.test(char));
}
export function skipHorizontalWhitespace(text: string, start: number): number {
let index = start;
while (index < text.length && (text[index] === " " || text[index] === "\t")) {
index += 1;
}
return index;
}
export function skipWhitespace(text: string, start: number): number {
let index = start;
while (index < text.length && /\s/.test(text[index] ?? "")) {
index += 1;
}
return index;
}
export function consumeLineBreak(text: string, start: number): number | null {
if (text[start] === "\r") {
return text[start + 1] === "\n" ? start + 2 : start + 1;
}
if (text[start] === "\n") {
return start + 1;
}
return null;
}
export function findJsonObjectEnd(
text: string,
start: number,
maxPayloadBytes?: number,
): number | null {
let depth = 0;
let inString = false;
let escaped = false;
for (let index = start; index < text.length; index += 1) {
if (maxPayloadBytes !== undefined && index + 1 - start > maxPayloadBytes) {
return null;
}
const char = text[index];
if (inString) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === '"') {
inString = false;
}
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === "{") {
depth += 1;
continue;
}
if (char === "}") {
depth -= 1;
if (depth === 0) {
return index + 1;
}
}
}
return null;
}
export function skipSerializedToolCallTrailingLineBreak(text: string, cursor: number): number {
const afterLineBreak = consumeLineBreak(text, cursor);
return afterLineBreak ?? cursor;
}
export function consumeJsonToolClosingMarker(text: string, cursor: number): number {
let markerStart = cursor;
while (markerStart < text.length && /\s/.test(text[markerStart] ?? "")) {
markerStart += 1;
}
const rest = text.slice(markerStart);
if (rest.startsWith(END_TOOL_REQUEST)) {
return skipSerializedToolCallTrailingLineBreak(text, markerStart + END_TOOL_REQUEST.length);
}
const bracketClose = /^\[\/[A-Za-z0-9_-]+\]/.exec(rest);
if (bracketClose) {
return skipSerializedToolCallTrailingLineBreak(text, markerStart + bracketClose[0].length);
}
if (rest.startsWith(HARMONY_CALL_MARKER)) {
return skipSerializedToolCallTrailingLineBreak(text, markerStart + HARMONY_CALL_MARKER.length);
}
return skipSerializedToolCallTrailingLineBreak(text, cursor);
}
export function findBracketedJsonPayloadStart(text: string): number | null {
if (!text.startsWith("[")) {
return null;
}
const close = text.indexOf("]");
if (close === -1) {
return null;
}
let cursor = close + 1;
cursor = skipHorizontalWhitespace(text, cursor);
cursor = skipSerializedToolCallTrailingLineBreak(text, cursor);
cursor = skipHorizontalWhitespace(text, cursor);
return text[cursor] === "{" ? cursor : null;
}
export function findHarmonyJsonPayloadStart(text: string): number | null {
let cursor = 0;
if (text.startsWith(HARMONY_CHANNEL_MARKER)) {
cursor = HARMONY_CHANNEL_MARKER.length;
}
const rest = text.slice(cursor);
const channel = ["commentary", "analysis", "final"].find((candidate) =>
rest.startsWith(candidate),
);
if (!channel) {
return null;
}
cursor += channel.length;
cursor = skipHorizontalWhitespace(text, cursor);
if (!text.slice(cursor).startsWith("to=")) {
return null;
}
cursor += "to=".length;
const nameStart = cursor;
while (isPlainTextToolNameChar(text[cursor])) {
cursor += 1;
}
if (cursor === nameStart) {
return null;
}
cursor = skipHorizontalWhitespace(text, cursor);
if (!text.slice(cursor).startsWith("code")) {
return null;
}
cursor += "code".length;
cursor = skipWhitespace(text, cursor);
if (text.slice(cursor).startsWith(HARMONY_MESSAGE_MARKER)) {
cursor = skipWhitespace(text, cursor + HARMONY_MESSAGE_MARKER.length);
}
return text[cursor] === "{" ? cursor : null;
}
export function startsWithAsciiMarkerIgnoreCase(
text: string,
cursor: number,
marker: string,
): boolean {
return text.slice(cursor, cursor + marker.length).toLowerCase() === marker;
}
export function indexOfAsciiMarkerIgnoreCase(text: string, marker: string, start: number): number {
let cursor = start;
while (cursor < text.length) {
const next = text.indexOf(marker[0] ?? "", cursor);
if (next === -1) {
return -1;
}
if (startsWithAsciiMarkerIgnoreCase(text, next, marker)) {
return next;
}
cursor = next + 1;
}
return -1;
}
export function findXmlishToolCallEnd(text: string): number | null {
let cursor: number;
const xmlFunction = /^<function=[A-Za-z0-9_.:-]+>/i.exec(text);
if (xmlFunction) {
cursor = xmlFunction[0].length;
} else {
const bracketed = /^\[(?:tool:)?[A-Za-z0-9_-]+\]/.exec(text);
if (!bracketed) {
return null;
}
cursor = bracketed[0].length;
cursor = skipHorizontalWhitespace(text, cursor);
cursor = skipSerializedToolCallTrailingLineBreak(text, cursor);
}
cursor = skipWhitespace(text, cursor);
if (!startsWithAsciiMarkerIgnoreCase(text, cursor, "<parameter=")) {
return null;
}
while (cursor < text.length) {
const parameterClose = indexOfAsciiMarkerIgnoreCase(text, "</parameter>", cursor);
if (parameterClose === -1) {
return null;
}
cursor = skipWhitespace(text, parameterClose + "</parameter>".length);
if (startsWithAsciiMarkerIgnoreCase(text, cursor, "</function>")) {
return skipSerializedToolCallTrailingLineBreak(text, cursor + "</function>".length);
}
if (!startsWithAsciiMarkerIgnoreCase(text, cursor, "<parameter=")) {
return skipSerializedToolCallTrailingLineBreak(text, cursor);
}
}
return null;
}

View File

@@ -0,0 +1,20 @@
export {
parseStandalonePlainTextToolCallBlocks,
stripPlainTextToolCallBlocks,
type PlainTextToolCallBlock,
type PlainTextToolCallParseOptions,
} from "./payload.js";
export {
normalizePlainTextToolCallStreamEvents,
scrubOverCapPlainTextToolCallMessage,
type PlainTextToolCallMessageNormalization,
type PlainTextToolCallNameMatcher,
type PlainTextToolCallStreamNormalizerOptions,
} from "./stream-normalizer.js";
export {
extractStandalonePlainTextToolCallText,
promoteStandalonePlainTextToolCallMessage,
type PlainTextToolCallPromotionOptions,
type PromotedPlainTextToolCallBlockFactory,
type ToolCallRepairNameResolver,
} from "./promote.js";

View File

@@ -0,0 +1,427 @@
import {
consumeLineBreak,
END_TOOL_REQUEST,
findJsonObjectEnd,
HARMONY_CALL_MARKER,
HARMONY_CHANNEL_MARKER,
HARMONY_MESSAGE_MARKER,
isPlainTextToolNameChar,
skipHorizontalWhitespace,
skipWhitespace,
} from "./grammar.js";
export type PlainTextToolCallBlock = {
arguments: Record<string, unknown>;
end: number;
name: string;
raw: string;
start: number;
};
export type PlainTextToolCallParseOptions = {
allowedToolNames?: Iterable<string>;
maxPayloadBytes?: number;
};
const DEFAULT_MAX_PLAIN_TEXT_TOOL_PAYLOAD_BYTES = 256_000;
type PlainTextToolCallOpening = {
allowsOptionalXmlishClose?: boolean;
end: number;
name: string;
requiresClosing: boolean;
};
function parseBracketOpening(text: string, start: number): PlainTextToolCallOpening | null {
if (text[start] !== "[") {
return null;
}
let cursor = start + 1;
if (text.startsWith("tool:", cursor)) {
cursor += "tool:".length;
const nameStart = cursor;
while (isPlainTextToolNameChar(text[cursor])) {
cursor += 1;
}
if (cursor === nameStart || text[cursor] !== "]") {
return null;
}
return {
allowsOptionalXmlishClose: true,
end: cursor + 1,
name: text.slice(nameStart, cursor),
requiresClosing: false,
};
}
const nameStart = cursor;
while (isPlainTextToolNameChar(text[cursor])) {
cursor += 1;
}
if (cursor === nameStart || text[cursor] !== "]") {
return null;
}
const name = text.slice(nameStart, cursor);
cursor += 1;
cursor = skipHorizontalWhitespace(text, cursor);
const afterLineBreak = consumeLineBreak(text, cursor);
if (afterLineBreak === null) {
return null;
}
return { end: afterLineBreak, name, requiresClosing: true };
}
function parseHarmonyOpening(text: string, start: number): PlainTextToolCallOpening | null {
let cursor = start;
if (text.startsWith(HARMONY_CHANNEL_MARKER, cursor)) {
cursor += HARMONY_CHANNEL_MARKER.length;
}
const channelStart = cursor;
while (/[A-Za-z_]/.test(text[cursor] ?? "")) {
cursor += 1;
}
const channel = text.slice(channelStart, cursor);
if (channel !== "commentary" && channel !== "analysis" && channel !== "final") {
return null;
}
cursor = skipHorizontalWhitespace(text, cursor);
if (!text.startsWith("to=", cursor)) {
return null;
}
cursor += 3;
const nameStart = cursor;
while (isPlainTextToolNameChar(text[cursor])) {
cursor += 1;
}
if (cursor === nameStart) {
return null;
}
const name = text.slice(nameStart, cursor);
cursor = skipHorizontalWhitespace(text, cursor);
if (!text.startsWith("code", cursor)) {
return null;
}
cursor += 4;
cursor = skipWhitespace(text, cursor);
if (text.startsWith(HARMONY_MESSAGE_MARKER, cursor)) {
cursor = skipWhitespace(text, cursor + HARMONY_MESSAGE_MARKER.length);
}
return { end: cursor, name, requiresClosing: false };
}
function parseXmlishFunctionOpening(text: string, start: number): PlainTextToolCallOpening | null {
const match = /^<function=([A-Za-z0-9_.:-]{1,120})>\s*/i.exec(text.slice(start));
if (!match?.[1]) {
return null;
}
return { end: start + match[0].length, name: match[1], requiresClosing: false };
}
function parseOpening(text: string, start: number): PlainTextToolCallOpening | null {
return parseBracketOpening(text, start) ?? parseHarmonyOpening(text, start);
}
function consumeJsonObject(
text: string,
start: number,
maxPayloadBytes: number,
): { end: number; value: Record<string, unknown> } | null {
const cursor = skipWhitespace(text, start);
if (text[cursor] !== "{") {
return null;
}
const end = findJsonObjectEnd(text, cursor, maxPayloadBytes);
if (end === null) {
return null;
}
const rawJson = text.slice(cursor, end);
try {
const parsed = JSON.parse(rawJson) as unknown;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return null;
}
return { end, value: parsed as Record<string, unknown> };
} catch {
return null;
}
}
function parseClosing(text: string, start: number, name: string): number | null {
const cursor = skipWhitespace(text, start);
if (text.startsWith(END_TOOL_REQUEST, cursor)) {
return cursor + END_TOOL_REQUEST.length;
}
const namedClosing = `[/${name}]`;
if (text.startsWith(namedClosing, cursor)) {
return cursor + namedClosing.length;
}
return null;
}
function parseOptionalHarmonyClosing(text: string, start: number): number {
const cursor = skipWhitespace(text, start);
if (text.startsWith(HARMONY_CALL_MARKER, cursor)) {
return cursor + HARMONY_CALL_MARKER.length;
}
return start;
}
function parsePlainTextToolCallBlockAt(
text: string,
start: number,
options?: PlainTextToolCallParseOptions,
): PlainTextToolCallBlock | null {
const opening = parseOpening(text, start);
if (!opening) {
return null;
}
const allowedToolNames = options?.allowedToolNames
? new Set(options.allowedToolNames)
: undefined;
if (allowedToolNames && !allowedToolNames.has(opening.name)) {
return null;
}
const payload = consumeJsonObject(
text,
opening.end,
options?.maxPayloadBytes ?? DEFAULT_MAX_PLAIN_TEXT_TOOL_PAYLOAD_BYTES,
);
if (!payload) {
return null;
}
const closingEnd = opening.requiresClosing
? parseClosing(text, payload.end, opening.name)
: parseOptionalHarmonyClosing(text, payload.end);
if (closingEnd === null) {
return null;
}
return {
arguments: payload.value,
end: closingEnd,
name: opening.name,
raw: text.slice(start, closingEnd),
start,
};
}
type XmlishParameterBlockBounds = {
closeStart: number;
end: number;
name: string;
payloadStart: number;
start: number;
};
function findXmlishParameterBlock(text: string, start: number): XmlishParameterBlockBounds | null {
const cursor = skipWhitespace(text, start);
const openMatch = /^<parameter=([A-Za-z0-9_.:-]{1,120})>/i.exec(text.slice(cursor));
if (!openMatch?.[1]) {
return null;
}
const payloadStart = cursor + openMatch[0].length;
const closeMatch = /<\/parameter>/i.exec(text.slice(payloadStart));
if (!closeMatch) {
return null;
}
const closeStart = payloadStart + closeMatch.index;
const closeEnd = closeStart + closeMatch[0].length;
return {
closeStart,
end: closeEnd,
name: openMatch[1],
payloadStart,
start: cursor,
};
}
function consumeXmlishParameterBlock(
text: string,
start: number,
maxPayloadBytes: number,
): { end: number; name: string; value: string } | null {
const bounds = findXmlishParameterBlock(text, start);
if (!bounds) {
return null;
}
if (bounds.end - bounds.start > maxPayloadBytes) {
return null;
}
return {
end: bounds.end,
name: bounds.name,
value: extractXmlishParameterValue(text, bounds.payloadStart, bounds.closeStart),
};
}
function extractXmlishParameterValue(text: string, start: number, end: number): string {
let payloadStart = start;
let payloadEnd = end;
const afterOpeningLineBreak = consumeLineBreak(text, payloadStart);
if (afterOpeningLineBreak !== null) {
payloadStart = afterOpeningLineBreak;
if (payloadEnd > payloadStart && text[payloadEnd - 1] === "\n") {
payloadEnd -= 1;
if (payloadEnd > payloadStart && text[payloadEnd - 1] === "\r") {
payloadEnd -= 1;
}
} else if (payloadEnd > payloadStart && text[payloadEnd - 1] === "\r") {
payloadEnd -= 1;
}
}
return text.slice(payloadStart, payloadEnd);
}
function consumeXmlishFunctionClose(text: string, start: number): number | null {
const cursor = skipWhitespace(text, start);
return text.slice(cursor).toLowerCase().startsWith("</function>")
? cursor + "</function>".length
: null;
}
function consumeOptionalXmlishFunctionClose(text: string, start: number): number {
return consumeXmlishFunctionClose(text, start) ?? start;
}
function parseXmlishPlainTextToolCallBlockEndAt(
text: string,
start: number,
options?: PlainTextToolCallParseOptions,
): number | null {
const opening = parseXmlishOpening(text, start);
if (!opening) {
return null;
}
const allowedToolNames = options?.allowedToolNames
? new Set(options.allowedToolNames)
: undefined;
if (allowedToolNames && !allowedToolNames.has(opening.name)) {
return null;
}
let cursor = opening.end;
let parameterCount = 0;
while (true) {
const parameter = findXmlishParameterBlock(text, cursor);
if (!parameter) {
break;
}
parameterCount += 1;
cursor = parameter.end;
}
if (parameterCount === 0) {
return null;
}
return opening.allowsOptionalXmlishClose
? consumeOptionalXmlishFunctionClose(text, cursor)
: consumeXmlishFunctionClose(text, cursor);
}
function parseXmlishOpening(text: string, start: number): PlainTextToolCallOpening | null {
return parseBracketOpening(text, start) ?? parseXmlishFunctionOpening(text, start);
}
function parseXmlishPlainTextToolCallBlockAt(
text: string,
start: number,
options?: PlainTextToolCallParseOptions,
): PlainTextToolCallBlock | null {
const opening = parseXmlishOpening(text, start);
if (!opening) {
return null;
}
const allowedToolNames = options?.allowedToolNames
? new Set(options.allowedToolNames)
: undefined;
if (allowedToolNames && !allowedToolNames.has(opening.name)) {
return null;
}
const maxPayloadBytes = options?.maxPayloadBytes ?? DEFAULT_MAX_PLAIN_TEXT_TOOL_PAYLOAD_BYTES;
const args: Record<string, unknown> = {};
let cursor = opening.end;
let parameterCount = 0;
while (true) {
const parameter = consumeXmlishParameterBlock(text, cursor, maxPayloadBytes);
if (!parameter) {
break;
}
if (parameter.end - opening.end > maxPayloadBytes) {
return null;
}
args[parameter.name] = parameter.value;
parameterCount += 1;
cursor = parameter.end;
}
if (parameterCount === 0) {
return null;
}
const end = opening.allowsOptionalXmlishClose
? consumeOptionalXmlishFunctionClose(text, cursor)
: consumeXmlishFunctionClose(text, cursor);
if (end === null) {
return null;
}
return {
arguments: args,
end,
name: opening.name,
raw: text.slice(start, end),
start,
};
}
export function parseStandalonePlainTextToolCallBlocks(
text: string,
options?: PlainTextToolCallParseOptions,
): PlainTextToolCallBlock[] | null {
const blocks: PlainTextToolCallBlock[] = [];
let cursor = skipWhitespace(text, 0);
while (cursor < text.length) {
const block =
parsePlainTextToolCallBlockAt(text, cursor, options) ??
parseXmlishPlainTextToolCallBlockAt(text, cursor, options);
if (!block) {
return null;
}
blocks.push(block);
cursor = skipWhitespace(text, block.end);
}
return blocks.length > 0 ? blocks : null;
}
export function stripPlainTextToolCallBlocks(text: string): string {
if (
!text ||
(!/\[(?:tool:)?[A-Za-z0-9_-]+\]/.test(text) &&
!/(?:^|\n)\s*(?:<\|channel\|>)?(?:commentary|analysis|final)\s+to=/.test(text) &&
!/(?:^|\n)\s*<function=[A-Za-z0-9_.:-]{1,120}>/i.test(text))
) {
return text;
}
let result = "";
let cursor = 0;
let index = 0;
while (index < text.length) {
const lineStart = index === 0 || text[index - 1] === "\n";
if (!lineStart) {
index += 1;
continue;
}
const blockStart = skipHorizontalWhitespace(text, index);
const block = parsePlainTextToolCallBlockAt(text, blockStart);
const blockEnd = block?.end ?? parseXmlishPlainTextToolCallBlockEndAt(text, blockStart);
if (blockEnd === null) {
index += 1;
continue;
}
result += text.slice(cursor, index);
cursor = blockEnd;
const afterBlockLineBreak = consumeLineBreak(text, cursor);
if (afterBlockLineBreak !== null) {
cursor = afterBlockLineBreak;
}
index = cursor;
}
result += text.slice(cursor);
return result;
}

View File

@@ -0,0 +1,257 @@
import { parseStandalonePlainTextToolCallBlocks, type PlainTextToolCallBlock } from "./payload.js";
export type ToolCallRepairNameResolver = (
rawName: string,
allowedToolNames: Set<string>,
) => string | null;
export type PromotedPlainTextToolCallBlockFactory = (
block: PlainTextToolCallBlock,
resolvedName: string,
) => Record<string, unknown>;
export type PlainTextToolCallPromotionOptions = {
allowedStopReasons?: ReadonlySet<unknown>;
allowedToolNames: Set<string>;
createToolCallBlock: PromotedPlainTextToolCallBlockFactory;
isRetainableNonTextBlock?: (block: Record<string, unknown>) => boolean;
message: unknown;
requireAssistantRole?: boolean;
resolveToolName?: ToolCallRepairNameResolver;
};
function asRecord(value: unknown): Record<string, unknown> | undefined {
return value && typeof value === "object" ? (value as Record<string, unknown>) : undefined;
}
function resolveExactToolName(rawName: string, allowedToolNames: Set<string>): string | null {
return allowedToolNames.has(rawName) ? rawName : null;
}
function createPromotedToolCallBlocks(
text: string,
options: PlainTextToolCallPromotionOptions,
): Record<string, unknown>[] | undefined {
const parsedBlocks = parseStandalonePlainTextToolCallBlocks(text);
if (!parsedBlocks) {
return undefined;
}
const resolveToolName = options.resolveToolName ?? resolveExactToolName;
const toolCalls: Record<string, unknown>[] = [];
for (const block of parsedBlocks) {
const resolvedName = resolveToolName(block.name, options.allowedToolNames);
if (!resolvedName) {
return undefined;
}
toolCalls.push(options.createToolCallBlock(block, resolvedName));
}
return toolCalls;
}
function createPromotedToolCallBlocksFromTextParts(
textParts: readonly string[],
options: PlainTextToolCallPromotionOptions,
): Record<string, unknown>[] | undefined {
const exactText = textParts.join("").trim();
if (!exactText) {
return [];
}
for (const text of createTextPartPromotionCandidates(textParts, exactText)) {
const toolCalls = createPromotedToolCallBlocks(text, options);
if (toolCalls) {
return toolCalls;
}
}
return undefined;
}
function createTextPartPromotionCandidates(
textParts: readonly string[],
exactText: string,
): string[] {
const repairedText = joinTextPartsWithStructuralLineBreaks(textParts).trim();
const newlineJoinedText = textParts.join("\n").trim();
return [...new Set([repairedText, exactText, newlineJoinedText].filter(Boolean))];
}
function joinTextPartsWithStructuralLineBreaks(textParts: readonly string[]): string {
let text = "";
for (const part of textParts) {
if (text && shouldInsertStructuralLineBreak(text, part)) {
text += "\n";
}
text += part;
}
return text;
}
function shouldInsertStructuralLineBreak(left: string, right: string): boolean {
if (!left || !right || /[\r\n]$/u.test(left) || /^\s/u.test(right)) {
return false;
}
const trimmedLeft = left.trimEnd();
return (
/<parameter=[A-Za-z0-9_.:-]{1,120}>$/iu.test(trimmedLeft) ||
/^\[[A-Za-z0-9_-]+\]$/u.test(trimmedLeft)
);
}
function shouldPromoteMessage(options: PlainTextToolCallPromotionOptions): boolean {
if (options.allowedToolNames.size === 0) {
return false;
}
const messageRecord = asRecord(options.message);
if (!messageRecord) {
return false;
}
if (options.requireAssistantRole && messageRecord.role !== "assistant") {
return false;
}
return !options.allowedStopReasons || options.allowedStopReasons.has(messageRecord.stopReason);
}
export function extractStandalonePlainTextToolCallText(params: {
allowOtherNonTextBlocks?: boolean;
allowedStopReasons?: ReadonlySet<unknown>;
isRetainableNonTextBlock?: (block: Record<string, unknown>) => boolean;
message: unknown;
requireAssistantRole?: boolean;
}): string | undefined {
const record = asRecord(params.message);
if (!record) {
return undefined;
}
if (params.requireAssistantRole && record.role !== "assistant") {
return undefined;
}
if (params.allowedStopReasons && !params.allowedStopReasons.has(record.stopReason)) {
return undefined;
}
const content = record.content;
if (typeof content === "string") {
const text = content.trim();
return text || undefined;
}
if (!Array.isArray(content)) {
return undefined;
}
const textParts: string[] = [];
for (const block of content) {
const blockRecord = asRecord(block);
if (!blockRecord) {
return undefined;
}
if (blockRecord.type === "text") {
if (typeof blockRecord.text !== "string") {
return undefined;
}
if (blockRecord.text.trim()) {
textParts.push(blockRecord.text);
}
continue;
}
if (params.isRetainableNonTextBlock?.(blockRecord) || params.allowOtherNonTextBlocks) {
continue;
}
return undefined;
}
const text = textParts.join("").trim();
return text || undefined;
}
export function promoteStandalonePlainTextToolCallMessage(
options: PlainTextToolCallPromotionOptions,
): Record<string, unknown> | undefined {
if (!shouldPromoteMessage(options)) {
return undefined;
}
const messageRecord = asRecord(options.message);
if (!messageRecord) {
return undefined;
}
const originalContent = messageRecord.content;
if (typeof originalContent === "string") {
const text = originalContent.trim();
if (!text) {
return undefined;
}
const toolCalls = createPromotedToolCallBlocks(text, options);
if (!toolCalls) {
return undefined;
}
return {
...messageRecord,
content: toolCalls,
stopReason: "toolUse",
};
}
if (!Array.isArray(originalContent)) {
return undefined;
}
const content: Array<Record<string, unknown>> = [];
let promotedTextBlock = false;
let textParts: string[] = [];
const flushTextParts = (): boolean | undefined => {
if (textParts.length === 0) {
return false;
}
const toolCalls = createPromotedToolCallBlocksFromTextParts(textParts, options);
textParts = [];
if (toolCalls?.length === 0) {
return false;
}
if (!toolCalls) {
return undefined;
}
content.push(...toolCalls);
return true;
};
for (const block of originalContent) {
const blockRecord = asRecord(block);
if (!blockRecord) {
return undefined;
}
if (blockRecord.type === "text") {
if (typeof blockRecord.text !== "string") {
return undefined;
}
if (blockRecord.text.trim()) {
textParts.push(blockRecord.text);
}
continue;
}
const promotedTextRun = flushTextParts();
if (promotedTextRun === undefined) {
return undefined;
}
promotedTextBlock ||= promotedTextRun;
if (options.isRetainableNonTextBlock?.(blockRecord)) {
content.push(blockRecord);
continue;
}
return undefined;
}
const promotedTrailingTextRun = flushTextParts();
if (promotedTrailingTextRun === undefined) {
return undefined;
}
promotedTextBlock ||= promotedTrailingTextRun;
if (!promotedTextBlock) {
return undefined;
}
return {
...messageRecord,
content,
stopReason: "toolUse",
};
}

File diff suppressed because it is too large Load Diff

2
pnpm-lock.yaml generated
View File

@@ -1855,6 +1855,8 @@ importers:
packages/web-content-core: {}
packages/tool-call-repair: {}
ui:
dependencies:
'@create-markdown/preview':

View File

@@ -1,5 +1,5 @@
import { stripPlainTextToolCallBlocks } from "../../../packages/tool-call-repair/src/index.js";
import { stripInboundMetadata } from "../../auto-reply/reply/strip-inbound-meta.js";
import { stripPlainTextToolCallBlocks } from "../../plugin-sdk/tool-payload.js";
import {
extractLeadingHttpStatus,
formatRawAssistantErrorForUi,

View File

@@ -1,13 +1,51 @@
import type { AgentMessage } from "openclaw/plugin-sdk/agent-core";
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import {
sanitizeOpenAIResponsesReplayForStream,
sanitizeReplayToolCallIdsForStream,
shouldApplyReplayToolCallIdSanitizer,
wrapStreamFnPromoteStandaloneTextToolCalls,
} from "./attempt.tool-call-normalization.js";
type AssistantMessage = Extract<AgentMessage, { role: "assistant" }>;
type ToolResultMessage = Extract<AgentMessage, { role: "toolResult" }>;
type FakeWrappedStream = {
result: () => Promise<unknown>;
[Symbol.asyncIterator]: () => AsyncIterator<unknown>;
};
function createFakeStream(params: {
events: unknown[];
resultMessage: unknown;
}): FakeWrappedStream {
return {
async result() {
return params.resultMessage;
},
[Symbol.asyncIterator]() {
return (async function* () {
for (const event of params.events) {
yield event;
}
})();
},
};
}
async function collectStreamEvents(stream: AsyncIterable<unknown>): Promise<unknown[]> {
const events: unknown[] = [];
for await (const event of stream) {
events.push(event);
}
return events;
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object") {
throw new Error(`expected ${label}`);
}
return value as Record<string, unknown>;
}
function requireAssistantMessage(message: AgentMessage | undefined): AssistantMessage {
if (!message || message.role !== "assistant") {
@@ -50,6 +88,731 @@ function toolResultSummary(message: AgentMessage | undefined) {
};
}
describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => {
it("promotes standalone serialized parameter XML text to structured tool calls", async () => {
const rawToolText = [
"[tool:exec]",
"<parameter=command>",
"cat /proc/mounts 2>/dev/null | head -20",
"</parameter>",
"</function>",
"",
"<function=exec>",
"<parameter=command>",
"find / -maxdepth 4 -type d 2>/dev/null | head -20",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [
{ type: "thinking", thinking: "Need to audit the mount." },
{ type: "text", text: rawToolText },
],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "start", partial: { content: [] } },
{
type: "text_start",
contentIndex: 1,
partial: { content: [{ type: "text", text: "" }] },
},
{ type: "text_delta", contentIndex: 1, delta: rawToolText },
{ type: "text_end", contentIndex: 1, content: rawToolText },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"start",
"toolcall_start",
"toolcall_delta",
"toolcall_start",
"toolcall_delta",
"done",
]);
expect(requireRecord(events.at(-1), "done").reason).toBe("toolUse");
expect(result.stopReason).toBe("toolUse");
const content = result.content as Array<Record<string, unknown>>;
expect(content).toHaveLength(3);
expect(content[0]).toEqual({ type: "thinking", thinking: "Need to audit the mount." });
expect(content[1]).toMatchObject({
type: "toolCall",
name: "exec",
arguments: { command: "cat /proc/mounts 2>/dev/null | head -20" },
partialArgs: '{"command":"cat /proc/mounts 2>/dev/null | head -20"}',
});
expect(String(content[1].id)).toMatch(/^call_[a-f0-9]{24}$/);
expect(content[2]).toMatchObject({
type: "toolCall",
name: "exec",
arguments: { command: "find / -maxdepth 4 -type d 2>/dev/null | head -20" },
});
});
it("preserves content indexes when promoting text before thinking", async () => {
const rawToolText = [
"[tool:exec]",
"<parameter=command>",
"pwd",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [
{ type: "text", text: rawToolText },
{ type: "thinking", thinking: "Need the current directory." },
],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
{
type: "thinking_delta",
contentIndex: 1,
delta: "Need the current directory.",
partial: {
content: [
{ type: "text", text: rawToolText },
{ type: "thinking", thinking: "Need the current directory." },
],
},
},
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"thinking_delta",
"toolcall_start",
"toolcall_delta",
"done",
]);
expect(requireRecord(events[0], "thinking event").contentIndex).toBe(1);
expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0);
expect((result.content as Array<Record<string, unknown>>).map((block) => block.type)).toEqual([
"toolCall",
"thinking",
]);
});
it("preserves intervening thinking when promoting multiple text blocks", async () => {
const firstRawToolText = [
"[tool:exec]",
"<parameter=command>",
"pwd",
"</parameter>",
"</function>",
].join("\n");
const secondRawToolText = [
"[tool:exec]",
"<parameter=command>",
"whoami",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [
{ type: "text", text: firstRawToolText },
{ type: "thinking", thinking: "Need one more check." },
{ type: "text", text: secondRawToolText },
],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: firstRawToolText },
{
type: "thinking_delta",
contentIndex: 1,
delta: "Need one more check.",
partial: {
content: [
{ type: "text", text: firstRawToolText },
{ type: "thinking", thinking: "Need one more check." },
{ type: "text", text: secondRawToolText },
],
},
},
{ type: "text_delta", contentIndex: 2, delta: secondRawToolText },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"thinking_delta",
"toolcall_start",
"toolcall_delta",
"toolcall_start",
"toolcall_delta",
"done",
]);
expect(requireRecord(events[0], "thinking event").contentIndex).toBe(1);
expect(requireRecord(events[1], "first toolcall start").contentIndex).toBe(0);
expect(requireRecord(events[3], "second toolcall start").contentIndex).toBe(2);
expect((result.content as Array<Record<string, unknown>>).map((block) => block.type)).toEqual([
"toolCall",
"thinking",
"toolCall",
]);
expect(requireRecord((result.content as unknown[])[0], "first tool call")).toMatchObject({
name: "exec",
arguments: { command: "pwd" },
});
expect(requireRecord((result.content as unknown[])[2], "second tool call")).toMatchObject({
name: "exec",
arguments: { command: "whoami" },
});
});
it("promotes serialized tool calls split across adjacent text blocks", async () => {
const resultMessage = {
role: "assistant",
content: [
{ type: "text", text: "[tool:exec]\n<parameter=command>\n" },
{ type: "text", text: "pwd\n</parameter>\n</function>" },
{ type: "thinking", thinking: "Checking location." },
],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: "[tool:exec]\n<parameter=command>\n" },
{ type: "text_delta", contentIndex: 1, delta: "pwd\n</parameter>\n</function>" },
{
type: "thinking_delta",
contentIndex: 2,
delta: "Checking location.",
partial: { content: resultMessage.content },
},
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"thinking_delta",
"toolcall_start",
"toolcall_delta",
"done",
]);
expect(requireRecord(events[0], "thinking event").contentIndex).toBe(2);
expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0);
expect((result.content as Array<Record<string, unknown>>).map((block) => block.type)).toEqual([
"toolCall",
"thinking",
]);
expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({
name: "exec",
arguments: { command: "pwd" },
});
});
it("buffers case-insensitive tool-name prefixes until final promotion", async () => {
const rawToolText = [
"[tool:read]",
"<parameter=path>",
"src/index.ts",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [{ type: "text", text: rawToolText }],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: "[tool:rea" },
{ type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:rea".length) },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["Read"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"toolcall_start",
"toolcall_delta",
"done",
]);
expect(result.stopReason).toBe("toolUse");
expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({
type: "toolCall",
name: "Read",
arguments: { path: "src/index.ts" },
});
});
it("buffers normalized alias tool-name prefixes until final promotion", async () => {
const rawToolText = [
"[tool:bash]",
"<parameter=command>",
"pwd",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [{ type: "text", text: rawToolText }],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: "[tool:ba" },
{ type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:ba".length) },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"toolcall_start",
"toolcall_delta",
"done",
]);
expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({
type: "toolCall",
name: "exec",
arguments: { command: "pwd" },
});
});
it("keeps possible tool-call text buffered across interleaved non-text events", async () => {
const rawToolText = [
"[tool:exec]",
"<parameter=command>",
"pwd",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [
{ type: "thinking", thinking: "Need shell state." },
{ type: "text", text: rawToolText },
],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 1, delta: rawToolText },
{
type: "thinking_delta",
contentIndex: 0,
delta: "Need shell state.",
partial: {
content: [
{ type: "thinking", thinking: "Need shell state." },
{ type: "text", text: rawToolText },
],
},
},
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"thinking_delta",
"toolcall_start",
"toolcall_delta",
"done",
]);
const thinkingEvent = requireRecord(events[0], "thinking event");
expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([
{ type: "thinking", thinking: "Need shell state." },
]);
expect(JSON.stringify(events)).not.toContain(rawToolText);
});
it("preserves interleaved event content indexes when buffered text is scrubbed first", async () => {
const rawToolText = [
"[tool:exec]",
"<parameter=command>",
"pwd",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [
{ type: "text", text: rawToolText },
{ type: "thinking", thinking: "Need shell state." },
],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
{
type: "thinking_delta",
contentIndex: 1,
delta: "Need shell state.",
partial: {
content: [
{ type: "text", text: rawToolText },
{ type: "thinking", thinking: "Need shell state." },
],
},
},
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"thinking_delta",
"toolcall_start",
"toolcall_delta",
"done",
]);
const thinkingEvent = requireRecord(events[0], "thinking event");
expect(thinkingEvent.contentIndex).toBe(1);
expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([
{ type: "text", text: "" },
{ type: "thinking", thinking: "Need shell state." },
]);
expect(JSON.stringify(events)).not.toContain(rawToolText);
});
it("closes the underlying stream iterator when consumers stop early", async () => {
const returnIterator = vi.fn(async () => ({ done: true, value: undefined }));
const nextIterator = vi
.fn()
.mockResolvedValueOnce({ done: false, value: { type: "start", partial: { content: [] } } })
.mockResolvedValue({ done: true, value: undefined });
const baseFn = vi.fn(() => ({
async result() {
return { role: "assistant", content: [], stopReason: "stop" };
},
[Symbol.asyncIterator]() {
return {
next: nextIterator,
return: returnIterator,
};
},
}));
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const iterator = stream[Symbol.asyncIterator]();
expect(await iterator.next()).toEqual({
done: false,
value: { type: "start", partial: { content: [] } },
});
await iterator.return?.();
expect(returnIterator).toHaveBeenCalledTimes(1);
});
it("flushes buffered text before terminal error events", async () => {
const rawToolText = "[tool:exec]";
const errorEvent = { type: "error", error: new Error("stream failed") };
const baseFn = vi.fn(() =>
createFakeStream({
events: [{ type: "text_delta", contentIndex: 0, delta: rawToolText }, errorEvent],
resultMessage: { role: "assistant", content: [], stopReason: "stop" },
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
expect(events).toEqual([
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
errorEvent,
]);
});
it("buffers split XML function markers until final promotion", async () => {
const rawToolText = [
"<function=exec>",
"<parameter=command>",
"pwd",
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [{ type: "text", text: rawToolText }],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: "<" },
{ type: "text_delta", contentIndex: 0, delta: rawToolText.slice(1) },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"toolcall_start",
"toolcall_delta",
"done",
]);
});
it("suppresses over-cap serialized XMLish text instead of flushing it", async () => {
const rawToolText = [
"[tool:exec]",
"<parameter=command>",
"x".repeat(256_001),
"</parameter>",
"</function>",
].join("\n");
const resultMessage = {
role: "assistant",
content: [{ type: "text", text: rawToolText }],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "start", partial: { content: [] } },
{
type: "text_start",
contentIndex: 0,
partial: { content: [{ type: "text", text: "" }] },
},
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
{
type: "thinking_delta",
contentIndex: 1,
delta: "still thinking",
partial: {
content: [
{ type: "text", text: rawToolText },
{ type: "thinking", thinking: "still thinking" },
],
},
},
{ type: "text_end", contentIndex: 0, content: rawToolText },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"start",
"thinking_delta",
"done",
]);
const thinkingEvent = requireRecord(events[1], "thinking event");
expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([
{ type: "text", text: "" },
{ type: "thinking", thinking: "still thinking" },
]);
const doneEvent = requireRecord(events[2], "done event");
expect(doneEvent.reason).toBe("stop");
expect(doneEvent.message).toMatchObject({
role: "assistant",
content: [],
stopReason: "stop",
});
expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" });
expect(JSON.stringify(events)).not.toContain("[tool:exec]");
expect(JSON.stringify(result)).not.toContain("[tool:exec]");
});
it("scrubs split over-cap serialized XMLish text blocks from done messages", async () => {
const rawToolTextParts = [
"[tool:exec]\n<parameter=command>",
["x".repeat(256_001), "</parameter>", "</function>"].join("\n"),
];
const resultMessage = {
role: "assistant",
content: rawToolTextParts.map((text) => ({ type: "text", text })),
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [{ type: "done", reason: "stop", message: resultMessage }],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
const result = requireRecord(await stream.result(), "result message");
expect(requireRecord(events[0], "done event").message).toMatchObject({
role: "assistant",
content: [],
stopReason: "stop",
});
expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" });
expect(JSON.stringify(events)).not.toContain("[tool:exec]");
expect(JSON.stringify(result)).not.toContain("</parameter>");
});
it("preserves visible suffix text after an over-cap JSON tool payload", async () => {
const visibleSuffix = "Visible answer after oversized JSON.";
const rawText = [`[tool:exec] {"command":"${"x".repeat(256_001)}"}`, visibleSuffix].join("\n");
const resultMessage = {
role: "assistant",
content: [{ type: "text", text: rawText }],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: rawText },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
"text_delta",
"done",
]);
const textEvent = requireRecord(events[0], "text event");
expect(String(textEvent.delta)).toBe(visibleSuffix);
expect(requireRecord(textEvent.partial, "text partial").content).toEqual([
{ type: "text", text: visibleSuffix },
]);
expect(JSON.stringify(events)).not.toContain("[tool:exec]");
});
it("does not buffer normal prose that starts like a final answer", async () => {
const resultMessage = {
role: "assistant",
content: [{ type: "text", text: "Finally, the audit is done." }],
stopReason: "stop",
};
const baseFn = vi.fn(() =>
createFakeStream({
events: [
{ type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." },
{ type: "done", reason: "stop", message: resultMessage },
],
resultMessage,
}),
);
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
const stream = (await Promise.resolve(
wrapped({} as never, {} as never, {} as never),
)) as FakeWrappedStream;
const events = await collectStreamEvents(stream);
expect(events).toEqual([
{ type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." },
{ type: "done", reason: "stop", message: resultMessage },
]);
});
});
describe("sanitizeReplayToolCallIdsForStream", () => {
it("skips strict stream id sanitization when provider policy opts out", () => {
expect(

View File

@@ -1,5 +1,15 @@
import { randomUUID } from "node:crypto";
import {
extractStandalonePlainTextToolCallText,
normalizePlainTextToolCallStreamEvents,
promoteStandalonePlainTextToolCallMessage as promotePlainTextToolCallMessage,
scrubOverCapPlainTextToolCallMessage,
type PlainTextToolCallBlock,
type PlainTextToolCallNameMatcher,
} from "../../../../packages/tool-call-repair/src/index.js";
import { visitObjectContentBlocks } from "../../../shared/message-content-blocks.js";
import { normalizeLowercaseStringOrEmpty } from "../../../shared/string-coerce.js";
import { normalizeStringEntries } from "../../../shared/string-normalization.js";
import {
downgradeOpenAIFunctionCallReasoningPairs,
downgradeOpenAIReasoningBlocks,
@@ -9,14 +19,13 @@ import {
} from "../../embedded-agent-helpers.js";
import type { AgentMessage, StreamFn } from "../../runtime/index.js";
import { sanitizeToolUseResultPairing } from "../../session-transcript-repair.js";
import type { MutableAssistantMessageEventStream } from "../../stream-compat.js";
import {
extractToolCallsFromAssistant,
extractToolResultIds,
sanitizeToolCallIdsForCloudCodeAssist,
type ToolCallIdMode,
} from "../../tool-call-id.js";
import { normalizeToolName } from "../../tool-policy.js";
import { couldNormalizeToolNamePrefixToAllowedTool, normalizeToolName } from "../../tool-policy.js";
import { shouldAllowProviderOwnedThinkingReplay } from "../../transcript-policy.js";
import type { TranscriptPolicy } from "../../transcript-policy.js";
import { wrapStreamObjectEvents } from "./stream-wrapper.js";
@@ -28,6 +37,7 @@ type UnknownToolLoopGuardState = {
count: number;
countedMessages: WeakSet<object>;
};
type AssistantStream = Awaited<ReturnType<StreamFn>>;
function resolveCaseInsensitiveAllowedToolName(
rawName: string,
@@ -94,10 +104,7 @@ function buildStructuredToolNameCandidates(rawName: string): string[] {
addCandidate(normalizedDelimiter);
addCandidate(normalizeToolName(normalizedDelimiter));
const segments = normalizedDelimiter
.split(".")
.map((segment) => segment.trim())
.filter(Boolean);
const segments = normalizeStringEntries(normalizedDelimiter.split("."));
if (segments.length > 1) {
for (let index = 1; index < segments.length; index += 1) {
const suffix = segments.slice(index).join(".");
@@ -849,11 +856,209 @@ function guardUnknownToolLoopInMessage(
return true;
}
type PromotedTextToolCallBlock = {
type: "toolCall";
id: string;
name: string;
arguments: Record<string, unknown>;
partialArgs: string;
};
function asRecord(value: unknown): Record<string, unknown> | undefined {
return value && typeof value === "object" ? (value as Record<string, unknown>) : undefined;
}
function createStandaloneTextToolCallId(): string {
return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`;
}
function createPromotedTextToolCallBlock(
block: PlainTextToolCallBlock,
name: string,
): PromotedTextToolCallBlock {
return {
type: "toolCall",
id: createStandaloneTextToolCallId(),
name,
arguments: block.arguments,
partialArgs: JSON.stringify(block.arguments),
};
}
function isRetainableNonVisibleBlock(block: Record<string, unknown>): boolean {
return block.type === "thinking" || block.type === "redacted_thinking";
}
const STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS = new Set<unknown>(["stop"]);
const STANDALONE_TEXT_TOOL_CALL_SCRUB_STOP_REASONS = new Set<unknown>(["stop", "length"]);
function extractStandaloneTextToolCallCandidateForStopReasons(
message: unknown,
allowedStopReasons: ReadonlySet<unknown>,
):
| {
text: string;
}
| undefined {
const text = extractStandalonePlainTextToolCallText({
allowedStopReasons,
isRetainableNonTextBlock: isRetainableNonVisibleBlock,
message,
requireAssistantRole: true,
});
return text ? { text } : undefined;
}
function promoteStandaloneTextToolCallMessage(
message: unknown,
allowedToolNames?: Set<string>,
): Record<string, unknown> | undefined {
if (!allowedToolNames) {
return undefined;
}
return promotePlainTextToolCallMessage({
allowedStopReasons: STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS,
allowedToolNames,
createToolCallBlock: createPromotedTextToolCallBlock,
isRetainableNonTextBlock: isRetainableNonVisibleBlock,
message,
requireAssistantRole: true,
resolveToolName: resolveExactAllowedToolName,
});
}
function createPromotedToolCallEvents(
message: Record<string, unknown>,
): Array<Record<string, unknown>> {
const content = Array.isArray(message.content) ? message.content : [];
const events: Array<Record<string, unknown>> = [];
content.forEach((block, contentIndex) => {
const record = asRecord(block);
if (record?.type !== "toolCall") {
return;
}
events.push({ type: "toolcall_start", contentIndex, partial: message });
events.push({
type: "toolcall_delta",
contentIndex,
delta: typeof record.partialArgs === "string" ? record.partialArgs : "{}",
partial: message,
});
});
return events;
}
function createStandaloneToolCallNameMatcher(
allowedToolNames: Set<string>,
): PlainTextToolCallNameMatcher {
return {
hasExactName: (name) => Boolean(resolveExactAllowedToolName(name, allowedToolNames)),
hasNamePrefix: (prefix) => couldNormalizeToolNamePrefixToAllowedTool(prefix, allowedToolNames),
};
}
function wrapStreamPromoteStandaloneTextToolCalls(
stream: AssistantStream,
allowedToolNames: Set<string>,
): AssistantStream {
const matcher = createStandaloneToolCallNameMatcher(allowedToolNames);
const normalizedMessages = new WeakMap<
object,
{ kind: "promoted" | "scrubbed"; message: Record<string, unknown> }
>();
const normalizeMessage = (
message: unknown,
): { kind: "promoted" | "scrubbed"; message: Record<string, unknown> } | undefined => {
if (!message || typeof message !== "object") {
return undefined;
}
const cached = normalizedMessages.get(message);
if (cached) {
return cached;
}
const promoted = promoteStandaloneTextToolCallMessage(message, allowedToolNames);
if (promoted) {
const result = { kind: "promoted" as const, message: promoted };
normalizedMessages.set(message, result);
return result;
}
const scrubbed = scrubOverCapPlainTextToolCallMessage({
candidateText: extractStandaloneTextToolCallCandidateForStopReasons(
message,
STANDALONE_TEXT_TOOL_CALL_SCRUB_STOP_REASONS,
)?.text,
matcher,
message,
});
if (scrubbed) {
const result = { kind: "scrubbed" as const, message: scrubbed };
normalizedMessages.set(message, result);
return result;
}
return undefined;
};
const originalResult = stream.result.bind(stream);
stream.result = async () => {
const message = await originalResult();
return (normalizeMessage(message)?.message ?? message) as Awaited<
ReturnType<typeof originalResult>
>;
};
const originalAsyncIterator = stream[Symbol.asyncIterator].bind(stream);
(stream as unknown as { [Symbol.asyncIterator]: () => AsyncIterator<unknown> })[
Symbol.asyncIterator
] = async function* () {
const source = {
[Symbol.asyncIterator]: originalAsyncIterator,
} as AsyncIterable<unknown>;
yield* normalizePlainTextToolCallStreamEvents(source, {
createPromotedToolCallEvents,
matcher,
normalizeDoneMessage: ({ message, reason }) => {
if (reason === "stop") {
return normalizeMessage(message);
}
const scrubbed = scrubOverCapPlainTextToolCallMessage({
candidateText: extractStandaloneTextToolCallCandidateForStopReasons(
message,
STANDALONE_TEXT_TOOL_CALL_SCRUB_STOP_REASONS,
)?.text,
matcher,
message,
});
return scrubbed ? { kind: "scrubbed", message: scrubbed } : undefined;
},
});
};
return stream;
}
export function wrapStreamFnPromoteStandaloneTextToolCalls(
baseFn: StreamFn,
allowedToolNames?: Set<string>,
): StreamFn {
if (!allowedToolNames || allowedToolNames.size === 0) {
return baseFn;
}
return (model, context, streamOptions) => {
const maybeStream = baseFn(model, context, streamOptions);
if (maybeStream && typeof maybeStream === "object" && "then" in maybeStream) {
return Promise.resolve(maybeStream).then((stream) =>
wrapStreamPromoteStandaloneTextToolCalls(stream, allowedToolNames),
);
}
return wrapStreamPromoteStandaloneTextToolCalls(maybeStream, allowedToolNames);
};
}
function wrapStreamTrimToolCallNames(
stream: MutableAssistantMessageEventStream,
stream: AssistantStream,
allowedToolNames?: Set<string>,
options?: { unknownToolThreshold?: number; state?: UnknownToolLoopGuardState },
): MutableAssistantMessageEventStream {
): AssistantStream {
const unknownToolGuardState = options?.state ?? {
count: 0,
countedMessages: new WeakSet<object>(),

View File

@@ -379,6 +379,7 @@ import {
sanitizeOpenAIResponsesReplayForStream,
sanitizeReplayToolCallIdsForStream,
shouldApplyReplayToolCallIdSanitizer,
wrapStreamFnPromoteStandaloneTextToolCalls,
wrapStreamFnSanitizeMalformedToolCalls,
wrapStreamFnTrimToolCallNames,
} from "./attempt.tool-call-normalization.js";
@@ -454,6 +455,7 @@ export {
wrapStreamFnRepairMalformedToolCallArguments,
} from "./attempt.tool-call-argument-repair.js";
export {
wrapStreamFnPromoteStandaloneTextToolCalls,
wrapStreamFnSanitizeMalformedToolCalls,
wrapStreamFnTrimToolCallNames,
} from "./attempt.tool-call-normalization.js";
@@ -2616,6 +2618,10 @@ export async function runEmbeddedAttempt(
transcriptPolicy,
params.provider,
);
activeSession.agent.streamFn = wrapStreamFnPromoteStandaloneTextToolCalls(
activeSession.agent.streamFn,
allowedToolNames,
);
activeSession.agent.streamFn = wrapStreamFnTrimToolCallNames(
activeSession.agent.streamFn,
allowedToolNames,

View File

@@ -23,6 +23,50 @@ export function normalizeToolName(name: string) {
return TOOL_NAME_ALIASES[normalized] ?? normalized;
}
export function couldNormalizeToolNamePrefixToAllowedTool(
prefix: string,
allowedToolNames: Set<string>,
): boolean {
const normalizedPrefix = normalizeLowercaseStringOrEmpty(prefix);
if (!normalizedPrefix) {
return false;
}
const allowed = new Set<string>();
for (const toolName of allowedToolNames) {
const normalizedToolName = normalizeToolName(toolName);
const foldedToolName = normalizeLowercaseStringOrEmpty(toolName);
if (normalizedToolName) {
allowed.add(normalizedToolName);
}
if (foldedToolName) {
allowed.add(foldedToolName);
}
if (
normalizedToolName.startsWith(normalizedPrefix) ||
foldedToolName.startsWith(normalizedPrefix)
) {
return true;
}
}
const resolvedPrefix = normalizeToolName(normalizedPrefix);
if (resolvedPrefix !== normalizedPrefix) {
for (const toolName of allowed) {
if (toolName.startsWith(resolvedPrefix)) {
return true;
}
}
}
for (const [alias, toolName] of Object.entries(TOOL_NAME_ALIASES)) {
if (alias.startsWith(normalizedPrefix) && allowed.has(toolName)) {
return true;
}
}
return false;
}
export function normalizeToolList(list?: string[]) {
if (!list) {
return [];

View File

@@ -3,6 +3,7 @@ import { uniqueStrings } from "../shared/string-normalization.js";
import { IMPLICIT_ALLOW_ALL_FROM_ALSO_ALLOW } from "./sandbox-tool-policy.js";
import { expandToolGroups, normalizeToolList, normalizeToolName } from "./tool-policy-shared.js";
export {
couldNormalizeToolNamePrefixToAllowedTool,
expandToolGroups,
normalizeToolList,
normalizeToolName,

View File

@@ -1,4 +1,5 @@
import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload";
import { stripPlainTextToolCallBlocks } from "../../../packages/tool-call-repair/src/index.js";
import { resolveSessionAgentId } from "../../agents/agent-scope.js";
import type { AgentToolResult } from "../../agents/runtime/index.js";
import {
@@ -30,7 +31,6 @@ import {
import type { OutboundMediaAccess } from "../../media/load-options.js";
import { getAgentScopedMediaLocalRoots } from "../../media/local-roots.js";
import { resolveAgentScopedOutboundMediaAccess } from "../../media/read-capability.js";
import { stripPlainTextToolCallBlocks } from "../../plugin-sdk/tool-payload.js";
import { hasPollCreationParams } from "../../poll-params.js";
import { resolvePollMaxSelections } from "../../polls.js";
import { resolveFirstBoundAccountId } from "../../routing/bound-account-read.js";

View File

@@ -11,7 +11,7 @@
* @see https://github.com/openclaw/openclaw/issues/18558
*/
import { stripPlainTextToolCallBlocks } from "../../plugin-sdk/tool-payload.js";
import { stripPlainTextToolCallBlocks } from "../../../packages/tool-call-repair/src/index.js";
const INTERNAL_RUNTIME_SCAFFOLDING_TAGS = ["system-reminder", "previous_response"] as const;
const INTERNAL_RUNTIME_SCAFFOLDING_TAG_PATTERN = INTERNAL_RUNTIME_SCAFFOLDING_TAGS.join("|");

File diff suppressed because it is too large Load Diff

View File

@@ -1,11 +1,18 @@
import { randomUUID } from "node:crypto";
import {
extractStandalonePlainTextToolCallText,
normalizePlainTextToolCallStreamEvents,
promoteStandalonePlainTextToolCallMessage,
scrubOverCapPlainTextToolCallMessage,
type PlainTextToolCallNameMatcher,
type PlainTextToolCallMessageNormalization,
} from "../../packages/tool-call-repair/src/index.js";
import type { StreamFn } from "../agents/runtime/index.js";
import { streamWithPayloadPatch } from "../llm/providers/stream-wrappers/stream-payload-utils.js";
import { streamSimple } from "../llm/stream.js";
import { createAssistantMessageEventStream } from "../llm/utils/event-stream.js";
import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js";
import type { ProviderWrapStreamFnContext } from "./plugin-entry.js";
import { parseStandalonePlainTextToolCallBlocks } from "./tool-payload.js";
export type ProviderStreamWrapperFactory =
| ((streamFn: StreamFn | undefined) => StreamFn | undefined)
@@ -41,164 +48,6 @@ function resolveContextToolNames(context: Parameters<StreamFn>[1]): Set<string>
return new Set(names);
}
function matchesLiteralPrefix(text: string, literal: string): boolean {
return literal.startsWith(text) || text.startsWith(literal);
}
function skipHorizontalWhitespace(text: string, start: number): number {
let cursor = start;
while (cursor < text.length && /[ \t]/.test(text[cursor] ?? "")) {
cursor += 1;
}
return cursor;
}
function matchesAnyToolNamePrefix(text: string, toolNames: Set<string>): boolean {
if (!text) {
return true;
}
for (const toolName of toolNames) {
if (toolName.startsWith(text) || text.startsWith(toolName)) {
return true;
}
}
return false;
}
function couldStillBeJsonPayload(text: string, start: number): boolean {
let cursor = start;
while (cursor < text.length && /\s/.test(text[cursor] ?? "")) {
cursor += 1;
}
return cursor >= text.length || text[cursor] === "{";
}
function couldStillBeBracketedToolCall(text: string, toolNames: Set<string>): boolean {
if (!text.startsWith("[")) {
return false;
}
const toolPrefix = "[tool:";
if (matchesLiteralPrefix(text, toolPrefix)) {
if (text.length <= toolPrefix.length) {
return true;
}
const nameStart = toolPrefix.length;
let cursor = nameStart;
while (cursor < text.length && text[cursor] !== "]") {
cursor += 1;
}
const name = text.slice(nameStart, cursor).trim();
if (!matchesAnyToolNamePrefix(name, toolNames)) {
return false;
}
if (cursor >= text.length) {
return true;
}
if (text[cursor] !== "]") {
return false;
}
return couldStillBeJsonPayload(text, cursor + 1);
}
let cursor = 1;
while (cursor < text.length && text[cursor] !== "\n" && text[cursor] !== "]") {
cursor += 1;
}
const firstLine = text.slice(1, cursor);
if (!matchesAnyToolNamePrefix(firstLine.trim(), toolNames)) {
return false;
}
if (cursor >= text.length) {
return true;
}
if (text[cursor] === "]") {
return couldStillBeJsonPayload(text, text[cursor + 1] === "\n" ? cursor + 2 : cursor + 1);
}
if (text[cursor] !== "\n") {
return false;
}
return couldStillBeJsonPayload(text, cursor + 1);
}
function couldStillBeHarmonyToolCall(text: string, toolNames: Set<string>): boolean {
const harmonyChannelPrefix = "<|channel|>";
let cursor = 0;
if (matchesLiteralPrefix(text, harmonyChannelPrefix)) {
if (text.length <= harmonyChannelPrefix.length) {
return true;
}
cursor = harmonyChannelPrefix.length;
}
const channelRest = text.slice(cursor);
const channelName = ["commentary", "analysis", "final"].find((marker) =>
matchesLiteralPrefix(channelRest, marker),
);
if (channelName) {
if (channelRest.length <= channelName.length) {
return true;
}
cursor += channelName.length;
} else if (cursor === 0) {
return false;
} else {
return false;
}
const constraintMarker = " to=";
const constraintRest = text.slice(cursor);
if (matchesLiteralPrefix(constraintRest, constraintMarker)) {
if (constraintRest.length <= constraintMarker.length) {
return true;
}
cursor += constraintMarker.length;
const nameStart = cursor;
while (cursor < text.length && text[cursor] !== " " && text[cursor] !== "\n") {
cursor += 1;
}
const name = text.slice(nameStart, cursor).trim();
if (!matchesAnyToolNamePrefix(name, toolNames)) {
return false;
}
}
cursor = skipHorizontalWhitespace(text, cursor);
if (cursor >= text.length) {
return true;
}
const codeMarker = "code";
const codeRest = text.slice(cursor);
if (matchesLiteralPrefix(codeRest, codeMarker)) {
if (codeRest.length <= codeMarker.length) {
return true;
}
cursor += codeMarker.length;
cursor = skipHorizontalWhitespace(text, cursor);
if (cursor >= text.length) {
return true;
}
}
const messageMarker = "<|message|>";
const messageRest = text.slice(cursor);
if (matchesLiteralPrefix(messageRest, messageMarker)) {
return true;
}
return text[cursor] === "{";
}
function couldStillBePlainTextToolCall(text: string, toolNames: Set<string>): boolean {
if (text.length > 256_000) {
return false;
}
const trimmed = text.trimStart();
return (
trimmed.length === 0 ||
couldStillBeBracketedToolCall(trimmed, toolNames) ||
couldStillBeHarmonyToolCall(trimmed, toolNames)
);
}
function createSyntheticToolCallId(): string {
return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`;
}
@@ -221,65 +70,18 @@ function promotePlainTextToolCalls(
toolNames: Set<string>,
): Record<string, unknown> | undefined {
const messageRecord = toRecord(message);
if (!messageRecord) {
return undefined;
}
if (!Array.isArray(messageRecord.content)) {
if (typeof messageRecord.content !== "string" || !messageRecord.content.trim()) {
return undefined;
}
const parsed = parseStandalonePlainTextToolCallBlocks(messageRecord.content, {
allowedToolNames: toolNames,
});
if (!parsed) {
return undefined;
}
return {
...messageRecord,
content: parsed.map(createPlainTextToolCallBlock),
stopReason: "toolUse",
};
}
if (
messageRecord.content.some((block) => toRecord(block)?.type === "toolCall") ||
messageRecord.content.length === 0
Array.isArray(messageRecord?.content) &&
messageRecord.content.some((block) => toRecord(block)?.type === "toolCall")
) {
return undefined;
}
let promoted = false;
const nextContent: Array<Record<string, unknown>> = [];
for (const block of messageRecord.content) {
const blockRecord = toRecord(block);
if (!blockRecord) {
return undefined;
}
if (blockRecord.type !== "text") {
nextContent.push(blockRecord);
continue;
}
const text = typeof blockRecord.text === "string" ? blockRecord.text : "";
if (!text.trim()) {
continue;
}
const parsed = parseStandalonePlainTextToolCallBlocks(text, {
allowedToolNames: toolNames,
});
if (!parsed) {
return undefined;
}
nextContent.push(...parsed.map(createPlainTextToolCallBlock));
promoted = true;
}
if (!promoted) {
return undefined;
}
return {
...messageRecord,
content: nextContent,
stopReason: "toolUse",
};
return promoteStandalonePlainTextToolCallMessage({
allowedToolNames: toolNames,
createToolCallBlock: (block, name) => createPlainTextToolCallBlock({ ...block, name }),
isRetainableNonTextBlock: () => true,
message,
});
}
function emitPromotedToolCallEvents(
@@ -302,6 +104,44 @@ function emitPromotedToolCallEvents(
});
}
function extractPlainTextToolCallCandidate(message: unknown): string | undefined {
return extractStandalonePlainTextToolCallText({
allowOtherNonTextBlocks: true,
message,
});
}
function createProviderToolNameMatcher(toolNames: Set<string>): PlainTextToolCallNameMatcher {
return {
hasExactName: (name) => toolNames.has(name),
hasNamePrefix: (prefix) => {
for (const toolName of toolNames) {
if (toolName.startsWith(prefix)) {
return true;
}
}
return false;
},
};
}
function normalizeProviderDoneMessage(
message: unknown,
toolNames: Set<string>,
matcher: PlainTextToolCallNameMatcher,
): PlainTextToolCallMessageNormalization {
const scrubbedMessage = scrubOverCapPlainTextToolCallMessage({
candidateText: extractPlainTextToolCallCandidate(message),
matcher,
message,
});
if (scrubbedMessage) {
return { kind: "scrubbed", message: scrubbedMessage };
}
const promotedMessage = promotePlainTextToolCalls(message, toolNames);
return promotedMessage ? { kind: "promoted", message: promotedMessage } : undefined;
}
function wrapPlainTextToolCallStream(
source: ReturnType<StreamFn>,
context: Parameters<StreamFn>[1],
@@ -310,12 +150,11 @@ function wrapPlainTextToolCallStream(
if (toolNames.size === 0) {
return source;
}
const matcher = createProviderToolNameMatcher(toolNames);
const output = createAssistantMessageEventStream();
const stream = output as unknown as { push(event: unknown): void; end(): void };
void (async () => {
const bufferedTextEvents: unknown[] = [];
let bufferedText = "";
let ended = false;
const endStream = () => {
if (!ended) {
@@ -323,54 +162,25 @@ function wrapPlainTextToolCallStream(
stream.end();
}
};
const flushBufferedTextEvents = () => {
for (const event of bufferedTextEvents.splice(0)) {
stream.push(event);
}
bufferedText = "";
};
try {
for await (const event of source as AsyncIterable<unknown>) {
const record = toRecord(event);
const type = typeof record?.type === "string" ? record.type : "";
if (type === "text_start" || type === "text_delta" || type === "text_end") {
bufferedTextEvents.push(event);
if (typeof record?.delta === "string") {
bufferedText += record.delta;
} else if (typeof record?.content === "string" && !bufferedText) {
bufferedText = record.content;
}
if (!couldStillBePlainTextToolCall(bufferedText, toolNames)) {
flushBufferedTextEvents();
}
continue;
}
if (type === "done") {
const promotedMessage = promotePlainTextToolCalls(record?.message, toolNames);
if (promotedMessage) {
bufferedTextEvents.splice(0);
bufferedText = "";
emitPromotedToolCallEvents(stream, promotedMessage);
stream.push({ ...record, reason: "toolUse", message: promotedMessage });
} else {
flushBufferedTextEvents();
stream.push(event);
}
endStream();
return;
}
flushBufferedTextEvents();
const normalizedEvents = normalizePlainTextToolCallStreamEvents(
source as AsyncIterable<unknown>,
{
createPromotedToolCallEvents: (message) => {
const events: unknown[] = [];
emitPromotedToolCallEvents({ push: (event: unknown) => events.push(event) }, message);
return events;
},
matcher,
normalizeDoneMessage: ({ message }) =>
normalizeProviderDoneMessage(message, toolNames, matcher),
stopAfterDone: true,
},
);
for await (const event of normalizedEvents) {
stream.push(event);
if (type === "error") {
endStream();
return;
}
}
flushBufferedTextEvents();
} catch (error) {
stream.push({
type: "error",

View File

@@ -121,6 +121,126 @@ describe("parseStandalonePlainTextToolCallBlocks", () => {
]);
});
it("parses serialized parameter XML tool calls", () => {
const firstRaw = [
"[tool:exec]",
"<parameter=command>",
'cat /proc/mounts 2>/dev/null | grep -i "libra|rav|openclaw" | head -20',
"</parameter>",
"</function>",
].join("\n");
const secondRaw = [
"<function=exec>",
"<parameter=command>",
'find / -maxdepth 4 -type d \\( -name "ravdb" -o -name "librav" \\) 2>/dev/null | head -20',
"</parameter>",
"</function>",
].join("\n");
const raw = [firstRaw, "", secondRaw].join("\n");
const blocks = parseStandalonePlainTextToolCallBlocks(raw, {
allowedToolNames: ["exec"],
});
expect(blocks).toEqual([
{
name: "exec",
arguments: {
command: 'cat /proc/mounts 2>/dev/null | grep -i "libra|rav|openclaw" | head -20',
},
start: 0,
end: firstRaw.length,
raw: firstRaw,
},
{
name: "exec",
arguments: {
command:
'find / -maxdepth 4 -type d \\( -name "ravdb" -o -name "librav" \\) 2>/dev/null | head -20',
},
start: firstRaw.length + 2,
end: raw.length,
raw: secondRaw,
},
]);
});
it("preserves whitespace inside serialized XML parameter values", () => {
const raw = [
"<function=write>",
"<parameter=content>",
" first line",
" second line",
"",
"</parameter>",
"</function>",
].join("\n");
const blocks = parseStandalonePlainTextToolCallBlocks(raw, {
allowedToolNames: ["write"],
});
expect(blocks?.[0]?.arguments).toEqual({
content: " first line\n second line\n",
});
});
it("rejects serialized XML parameter calls without a function close", () => {
const raw = ["<function=exec>", "<parameter=command>", "pwd", "</parameter>"].join("\n");
expect(
parseStandalonePlainTextToolCallBlocks(raw, {
allowedToolNames: ["exec"],
}),
).toBeNull();
});
it("parses legacy tool-prefixed XML parameter calls without a function close", () => {
const raw = ["[tool:exec]", "<parameter=command>", "pwd", "</parameter>"].join("\n");
expect(
parseStandalonePlainTextToolCallBlocks(raw, {
allowedToolNames: ["exec"],
}),
).toEqual([
{
arguments: { command: "pwd" },
end: raw.length,
name: "exec",
raw,
start: 0,
},
]);
});
it("finds XML parameter close tags without lowercased string offsets", () => {
const dottedCapitalI = "\u0130";
const raw = [
"<function=write>",
"<parameter=content>",
dottedCapitalI,
"</parameter>",
"</function>",
].join("\n");
const blocks = parseStandalonePlainTextToolCallBlocks(raw, {
allowedToolNames: ["write"],
});
expect(blocks?.[0]?.arguments).toEqual({ content: dottedCapitalI });
});
it("rejects XML parameter blocks whose cumulative payload exceeds the cap", () => {
const firstParameter = ["<parameter=first>", "alpha", "</parameter>"].join("\n");
const secondParameter = ["<parameter=second>", "beta", "</parameter>"].join("\n");
const raw = ["<function=write>", firstParameter, secondParameter, "</function>"].join("\n");
const maxPayloadBytes = Math.max(firstParameter.length, secondParameter.length) + 1;
expect(
parseStandalonePlainTextToolCallBlocks(raw, {
allowedToolNames: ["write"],
maxPayloadBytes,
}),
).toBeNull();
});
it("respects allowed tool names for Harmony calls", () => {
const blocks = parseStandalonePlainTextToolCallBlocks(
'commentary to=write code {"path":"/tmp/file.txt","content":"x"}',
@@ -170,6 +290,7 @@ describe("stripPlainTextToolCallBlocks", () => {
"<parameter=command>",
'cat /proc/mounts 2>/dev/null | grep -i "libra|rav|openclaw" | head -20',
"</parameter>",
"</function>",
"",
"<function=exec>",
"<parameter=command>",
@@ -184,4 +305,59 @@ describe("stripPlainTextToolCallBlocks", () => {
),
).toBe("before\n\nafter");
});
it("keeps legacy bracketed XML parameter blocks scrubbed", () => {
expect(
stripPlainTextToolCallBlocks(
[
"before",
"[exec]",
"<parameter=command>",
"pwd",
"</parameter>",
"</function>",
"after",
].join("\n"),
),
).toBe("before\nafter");
});
it("preserves incomplete XML parameter blocks when stripping visible text", () => {
const text = ["before", "[exec]", "<parameter=command>", "pwd", "</parameter>", "after"].join(
"\n",
);
expect(stripPlainTextToolCallBlocks(text)).toBe(text);
});
it("strips legacy tool-prefixed XML parameter blocks without a function close", () => {
expect(
stripPlainTextToolCallBlocks(
["before", "[tool:exec]", "<parameter=command>", "pwd", "</parameter>", "after"].join("\n"),
),
).toBe("before\nafter");
});
it("strips oversized XML parameter tool calls without promoting them", () => {
const largeValue = "x".repeat(140_000);
const block = [
"<function=write>",
"<parameter=first>",
largeValue,
"</parameter>",
"<parameter=second>",
largeValue,
"</parameter>",
"</function>",
].join("\n");
expect(
parseStandalonePlainTextToolCallBlocks(block, {
allowedToolNames: ["write"],
}),
).toBeNull();
expect(stripPlainTextToolCallBlocks(["before", block, "after"].join("\n"))).toBe(
"before\nafter",
);
});
});

View File

@@ -1,3 +1,32 @@
import {
parseStandalonePlainTextToolCallBlocks as parseStandaloneRepairToolCallBlocks,
stripPlainTextToolCallBlocks as stripRepairToolCallBlocks,
} from "../../packages/tool-call-repair/src/index.js";
export type PlainTextToolCallBlock = {
arguments: Record<string, unknown>;
end: number;
name: string;
raw: string;
start: number;
};
export type PlainTextToolCallParseOptions = {
allowedToolNames?: Iterable<string>;
maxPayloadBytes?: number;
};
export function parseStandalonePlainTextToolCallBlocks(
text: string,
options?: PlainTextToolCallParseOptions,
): PlainTextToolCallBlock[] | null {
return parseStandaloneRepairToolCallBlocks(text, options);
}
export function stripPlainTextToolCallBlocks(text: string): string {
return stripRepairToolCallBlocks(text);
}
type ToolPayloadTextBlock = {
type: "text";
text: string;
@@ -41,378 +70,3 @@ export function extractToolPayload(result: ToolPayloadCarrier | null | undefined
return text;
}
}
export type PlainTextToolCallBlock = {
arguments: Record<string, unknown>;
end: number;
name: string;
raw: string;
start: number;
};
export type PlainTextToolCallParseOptions = {
allowedToolNames?: Iterable<string>;
maxPayloadBytes?: number;
};
const DEFAULT_MAX_PLAIN_TEXT_TOOL_PAYLOAD_BYTES = 256_000;
const END_TOOL_REQUEST = "[END_TOOL_REQUEST]";
const HARMONY_CHANNEL_MARKER = "<|channel|>";
const HARMONY_MESSAGE_MARKER = "<|message|>";
const HARMONY_CALL_MARKER = "<|call|>";
const XMLISH_PARAMETER_CLOSE = "</parameter>";
type PlainTextToolCallOpening = {
end: number;
name: string;
requiresClosing: boolean;
};
function isToolNameChar(char: string | undefined): boolean {
return Boolean(char && /[A-Za-z0-9_-]/.test(char));
}
function skipHorizontalWhitespace(text: string, start: number): number {
let index = start;
while (index < text.length && (text[index] === " " || text[index] === "\t")) {
index += 1;
}
return index;
}
function skipWhitespace(text: string, start: number): number {
let index = start;
while (index < text.length && /\s/.test(text[index] ?? "")) {
index += 1;
}
return index;
}
function consumeLineBreak(text: string, start: number): number | null {
if (text[start] === "\r") {
return text[start + 1] === "\n" ? start + 2 : start + 1;
}
if (text[start] === "\n") {
return start + 1;
}
return null;
}
function parseBracketOpening(text: string, start: number): PlainTextToolCallOpening | null {
if (text[start] !== "[") {
return null;
}
let cursor = start + 1;
if (text.startsWith("tool:", cursor)) {
cursor += "tool:".length;
const nameStart = cursor;
while (isToolNameChar(text[cursor])) {
cursor += 1;
}
if (cursor === nameStart || text[cursor] !== "]") {
return null;
}
return { end: cursor + 1, name: text.slice(nameStart, cursor), requiresClosing: false };
}
const nameStart = cursor;
while (isToolNameChar(text[cursor])) {
cursor += 1;
}
if (cursor === nameStart || text[cursor] !== "]") {
return null;
}
const name = text.slice(nameStart, cursor);
cursor += 1;
cursor = skipHorizontalWhitespace(text, cursor);
const afterLineBreak = consumeLineBreak(text, cursor);
if (afterLineBreak === null) {
return null;
}
return { end: afterLineBreak, name, requiresClosing: true };
}
function parseHarmonyOpening(text: string, start: number): PlainTextToolCallOpening | null {
let cursor = start;
if (text.startsWith(HARMONY_CHANNEL_MARKER, cursor)) {
cursor += HARMONY_CHANNEL_MARKER.length;
}
const channelStart = cursor;
while (/[A-Za-z_]/.test(text[cursor] ?? "")) {
cursor += 1;
}
const channel = text.slice(channelStart, cursor);
if (channel !== "commentary" && channel !== "analysis" && channel !== "final") {
return null;
}
cursor = skipHorizontalWhitespace(text, cursor);
if (!text.startsWith("to=", cursor)) {
return null;
}
cursor += 3;
const nameStart = cursor;
while (isToolNameChar(text[cursor])) {
cursor += 1;
}
if (cursor === nameStart) {
return null;
}
const name = text.slice(nameStart, cursor);
cursor = skipHorizontalWhitespace(text, cursor);
if (!text.startsWith("code", cursor)) {
return null;
}
cursor += 4;
cursor = skipWhitespace(text, cursor);
if (text.startsWith(HARMONY_MESSAGE_MARKER, cursor)) {
cursor = skipWhitespace(text, cursor + HARMONY_MESSAGE_MARKER.length);
}
return { end: cursor, name, requiresClosing: false };
}
function parseXmlishFunctionOpening(text: string, start: number): PlainTextToolCallOpening | null {
const match = /^<function=([A-Za-z0-9_.:-]{1,120})>\s*/i.exec(text.slice(start));
if (!match?.[1]) {
return null;
}
return { end: start + match[0].length, name: match[1], requiresClosing: false };
}
function parseOpening(text: string, start: number): PlainTextToolCallOpening | null {
return parseBracketOpening(text, start) ?? parseHarmonyOpening(text, start);
}
function consumeJsonObject(
text: string,
start: number,
maxPayloadBytes: number,
): { end: number; value: Record<string, unknown> } | null {
const cursor = skipWhitespace(text, start);
if (text[cursor] !== "{") {
return null;
}
let depth = 0;
let inString = false;
let escaped = false;
for (let index = cursor; index < text.length; index += 1) {
const char = text[index];
if (index + 1 - cursor > maxPayloadBytes) {
return null;
}
if (inString) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === '"') {
inString = false;
}
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === "{") {
depth += 1;
} else if (char === "}") {
depth -= 1;
if (depth === 0) {
const rawJson = text.slice(cursor, index + 1);
try {
const parsed = JSON.parse(rawJson) as unknown;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return null;
}
return { end: index + 1, value: parsed as Record<string, unknown> };
} catch {
return null;
}
}
}
}
return null;
}
function parseClosing(text: string, start: number, name: string): number | null {
const cursor = skipWhitespace(text, start);
if (text.startsWith(END_TOOL_REQUEST, cursor)) {
return cursor + END_TOOL_REQUEST.length;
}
const namedClosing = `[/${name}]`;
if (text.startsWith(namedClosing, cursor)) {
return cursor + namedClosing.length;
}
return null;
}
function parseOptionalHarmonyClosing(text: string, start: number): number {
const cursor = skipWhitespace(text, start);
if (text.startsWith(HARMONY_CALL_MARKER, cursor)) {
return cursor + HARMONY_CALL_MARKER.length;
}
return start;
}
function parsePlainTextToolCallBlockAt(
text: string,
start: number,
options?: PlainTextToolCallParseOptions,
): PlainTextToolCallBlock | null {
const opening = parseOpening(text, start);
if (!opening) {
return null;
}
const allowedToolNames = options?.allowedToolNames
? new Set(options.allowedToolNames)
: undefined;
if (allowedToolNames && !allowedToolNames.has(opening.name)) {
return null;
}
const payload = consumeJsonObject(
text,
opening.end,
options?.maxPayloadBytes ?? DEFAULT_MAX_PLAIN_TEXT_TOOL_PAYLOAD_BYTES,
);
if (!payload) {
return null;
}
const closingEnd = opening.requiresClosing
? parseClosing(text, payload.end, opening.name)
: parseOptionalHarmonyClosing(text, payload.end);
if (closingEnd === null) {
return null;
}
return {
arguments: payload.value,
end: closingEnd,
name: opening.name,
raw: text.slice(start, closingEnd),
start,
};
}
function consumeXmlishParameterBlock(
text: string,
start: number,
maxPayloadBytes: number,
): number | null {
const cursor = skipWhitespace(text, start);
const openMatch = /^<parameter=[A-Za-z0-9_.:-]{1,120}>\s*/i.exec(text.slice(cursor));
if (!openMatch) {
return null;
}
const payloadStart = cursor + openMatch[0].length;
const closeStart = text.toLowerCase().indexOf(XMLISH_PARAMETER_CLOSE, payloadStart);
if (closeStart === -1 || closeStart + XMLISH_PARAMETER_CLOSE.length - cursor > maxPayloadBytes) {
return null;
}
return closeStart + XMLISH_PARAMETER_CLOSE.length;
}
function consumeXmlishParameterBlocks(
text: string,
start: number,
maxPayloadBytes: number,
): number | null {
let cursor = start;
let consumed = false;
while (true) {
const next = consumeXmlishParameterBlock(text, cursor, maxPayloadBytes);
if (next === null) {
break;
}
if (next - start > maxPayloadBytes) {
return null;
}
cursor = next;
consumed = true;
}
return consumed ? cursor : null;
}
function consumeOptionalXmlishFunctionClose(text: string, start: number): number {
const cursor = skipWhitespace(text, start);
return text.slice(cursor).toLowerCase().startsWith("</function>")
? cursor + "</function>".length
: start;
}
function parseXmlishPlainTextToolCallBlockEndAt(
text: string,
start: number,
options?: PlainTextToolCallParseOptions,
): number | null {
const opening = parseBracketOpening(text, start) ?? parseXmlishFunctionOpening(text, start);
if (!opening) {
return null;
}
const allowedToolNames = options?.allowedToolNames
? new Set(options.allowedToolNames)
: undefined;
if (allowedToolNames && !allowedToolNames.has(opening.name)) {
return null;
}
const payloadEnd = consumeXmlishParameterBlocks(
text,
opening.end,
options?.maxPayloadBytes ?? DEFAULT_MAX_PLAIN_TEXT_TOOL_PAYLOAD_BYTES,
);
if (payloadEnd === null) {
return null;
}
return consumeOptionalXmlishFunctionClose(text, payloadEnd);
}
export function parseStandalonePlainTextToolCallBlocks(
text: string,
options?: PlainTextToolCallParseOptions,
): PlainTextToolCallBlock[] | null {
const blocks: PlainTextToolCallBlock[] = [];
let cursor = skipWhitespace(text, 0);
while (cursor < text.length) {
const block = parsePlainTextToolCallBlockAt(text, cursor, options);
if (!block) {
return null;
}
blocks.push(block);
cursor = skipWhitespace(text, block.end);
}
return blocks.length > 0 ? blocks : null;
}
export function stripPlainTextToolCallBlocks(text: string): string {
if (
!text ||
(!/\[(?:tool:)?[A-Za-z0-9_-]+\]/.test(text) &&
!/(?:^|\n)\s*(?:<\|channel\|>)?(?:commentary|analysis|final)\s+to=/.test(text) &&
!/(?:^|\n)\s*<function=[A-Za-z0-9_.:-]{1,120}>/i.test(text))
) {
return text;
}
let result = "";
let cursor = 0;
let index = 0;
while (index < text.length) {
const lineStart = index === 0 || text[index - 1] === "\n";
if (!lineStart) {
index += 1;
continue;
}
const blockStart = skipHorizontalWhitespace(text, index);
const block = parsePlainTextToolCallBlockAt(text, blockStart);
const blockEnd = block?.end ?? parseXmlishPlainTextToolCallBlockEndAt(text, blockStart);
if (blockEnd === null) {
index += 1;
continue;
}
result += text.slice(cursor, index);
cursor = blockEnd;
const afterBlockLineBreak = consumeLineBreak(text, cursor);
if (afterBlockLineBreak !== null) {
cursor = afterBlockLineBreak;
}
index = cursor;
}
result += text.slice(cursor);
return result;
}

View File

@@ -1 +1 @@
export { stripPlainTextToolCallBlocks } from "../../plugin-sdk/tool-payload.js";
export { stripPlainTextToolCallBlocks } from "../../../packages/tool-call-repair/src/index.js";