diff --git a/extensions/llama-cpp/src/inference-provider.test.ts b/extensions/llama-cpp/src/inference-provider.test.ts index c18d027fd74d..3ceed2839171 100644 --- a/extensions/llama-cpp/src/inference-provider.test.ts +++ b/extensions/llama-cpp/src/inference-provider.test.ts @@ -235,6 +235,194 @@ describe("llama.cpp inference provider", () => { maxTokens: 2048, customStopTriggers: ["END"], }); + expect(mocks.generateResponse.mock.calls[0]?.[1]).not.toHaveProperty("onResponseChunk"); + }); + + it("streams reasoning and text before a result-only native tool call", async () => { + mocks.generateResponse.mockImplementationOnce(async (_history, options) => { + options.onResponseChunk({ + type: "segment", + segmentType: "thought", + text: "First ", + tokens: [1], + }); + options.onResponseChunk({ + type: "segment", + segmentType: "thought", + text: "reason.", + tokens: [2], + }); + options.onTextChunk("Answer."); + return { + response: "Answer.", + functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }], + metadata: { stopReason: "functionCalls" }, + }; + }); + + const events = await collectEvents( + await createLlamaCppStreamFn({})( + { ...model, reasoning: true }, + { + messages: [{ role: "user", content: "Why?", timestamp: 1 }], + tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }], + }, + ), + ); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "thinking_start", + "thinking_delta", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_end", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(events.at(-1)).toMatchObject({ + type: "done", + message: { + content: [ + { type: "thinking", thinking: "First reason." }, + { type: "text", text: "Answer." }, + { type: "toolCall", name: "weather", arguments: { city: "Paris" } }, + ], + }, + }); + expect(events.find((event) => event.type === "thinking_delta")).toMatchObject({ + contentIndex: 0, + partial: { content: [{ type: "thinking", thinking: "First " }] }, + }); + expect(events.find((event) => event.type === "toolcall_start")).toMatchObject({ + contentIndex: 2, + }); + }); + + it("closes native reasoning before streaming a completed tool call", async () => { + mocks.generateResponse.mockImplementationOnce(async (_history, options) => { + options.onResponseChunk({ + type: "segment", + segmentType: "thought", + text: "Need current weather.", + tokens: [1], + }); + options.onFunctionCallParamsChunk({ + callIndex: 0, + functionName: "weather", + paramsChunk: '{"city":"Paris"}', + done: true, + }); + return { + response: "", + functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }], + metadata: { stopReason: "functionCalls" }, + }; + }); + + const events = await collectEvents( + await createLlamaCppStreamFn({})( + { ...model, reasoning: true }, + { + messages: [{ role: "user", content: "Weather?", timestamp: 1 }], + tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }], + }, + ), + ); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "thinking_start", + "thinking_delta", + "thinking_end", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(events.find((event) => event.type === "toolcall_start")).toMatchObject({ + contentIndex: 1, + }); + expect(events.at(-1)).toMatchObject({ + type: "done", + reason: "toolUse", + message: { + content: [ + { type: "thinking", thinking: "Need current weather." }, + { type: "toolCall", name: "weather", arguments: { city: "Paris" } }, + ], + }, + }); + }); + + it("opens a new indexed block for every thought segment after visible text", async () => { + mocks.generateResponse.mockImplementationOnce(async (_history, options) => { + options.onResponseChunk({ + type: "segment", + segmentType: "thought", + text: "First thought.", + tokens: [1], + segmentEndTime: new Date(1), + }); + options.onTextChunk("First answer."); + options.onResponseChunk({ + type: "segment", + segmentType: "thought", + text: "Second thought.", + tokens: [2], + segmentEndTime: new Date(2), + }); + options.onTextChunk("Second answer."); + return { + response: "First answer.Second answer.", + functionCalls: undefined, + metadata: { stopReason: "eogToken" }, + }; + }); + + const events = await collectEvents( + await createLlamaCppStreamFn({})( + { ...model, reasoning: true }, + { messages: [{ role: "user", content: "Reason twice", timestamp: 1 }] }, + ), + ); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "thinking_start", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_end", + "thinking_start", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_end", + "done", + ]); + expect( + events + .filter((event) => event.type === "thinking_start" || event.type === "text_start") + .map((event) => event.contentIndex), + ).toEqual([0, 1, 2, 3]); + expect(events.at(-1)).toMatchObject({ + type: "done", + message: { + content: [ + { type: "thinking", thinking: "First thought." }, + { type: "text", text: "First answer." }, + { type: "thinking", thinking: "Second thought." }, + { type: "text", text: "Second answer." }, + ], + }, + }); }); it("builds a JSON Schema grammar for tool-free responseFormat requests", async () => { @@ -327,7 +515,7 @@ describe("llama.cpp inference provider", () => { expect(mocks.generateResponse.mock.calls[0]?.[1]).not.toHaveProperty("grammar"); }); - it("emits native function calls in the final assistant message", async () => { + it("streams the complete lifecycle for native function calls", async () => { mocks.generateResponse.mockResolvedValueOnce({ response: "", functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }], @@ -346,7 +534,31 @@ describe("llama.cpp inference provider", () => { const events = await collectEvents(stream); - expect(events.map((event) => event.type)).toEqual(["done"]); + expect(events.map((event) => event.type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + const toolCallStart = events[1]; + const toolCallDelta = events[2]; + const toolCallEnd = events[3]; + expect(toolCallStart).toMatchObject({ + type: "toolcall_start", + contentIndex: 0, + partial: { content: [{ type: "toolCall", name: "weather", arguments: {} }] }, + }); + expect(toolCallDelta).toMatchObject({ + type: "toolcall_delta", + contentIndex: 0, + delta: '{"city":"Paris"}', + }); + expect(toolCallEnd).toMatchObject({ + type: "toolcall_end", + contentIndex: 0, + toolCall: { name: "weather", arguments: { city: "Paris" } }, + }); expect(events.at(-1)).toMatchObject({ type: "done", reason: "toolUse", @@ -364,6 +576,217 @@ describe("llama.cpp inference provider", () => { expect(mocks.llama.createGrammarForJsonSchema).not.toHaveBeenCalled(); }); + it("streams split native arguments with stable ids after mixed text", async () => { + mocks.generateResponse.mockImplementationOnce(async (_history, options) => { + options.onTextChunk("Checking both."); + options.onFunctionCallParamsChunk({ + callIndex: 0, + functionName: "weather", + paramsChunk: '{"city":', + done: false, + }); + options.onFunctionCallParamsChunk({ + callIndex: 0, + functionName: "weather", + paramsChunk: '"Paris"}', + done: true, + }); + options.onFunctionCallParamsChunk({ + callIndex: 1, + functionName: "calendar", + paramsChunk: '{"day":"today"}', + done: true, + }); + return { + response: "Checking both.", + functionCalls: [ + { functionName: "weather", params: { city: "Paris" }, raw: [] }, + { functionName: "calendar", params: { day: "today" }, raw: [] }, + ], + metadata: { stopReason: "functionCalls" }, + }; + }); + + const stream = await createLlamaCppStreamFn({})(model, { + messages: [{ role: "user", content: "Check both", timestamp: 1 }], + tools: [ + { name: "weather", description: "Weather", parameters: { type: "object" } }, + { name: "calendar", description: "Calendar", parameters: { type: "object" } }, + ], + }); + const events = await collectEvents(stream); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "text_start", + "text_delta", + "text_end", + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "toolcall_end", + "done", + ]); + const toolDeltas = events.filter((event) => event.type === "toolcall_delta"); + expect( + events.find((event) => event.type === "toolcall_start")?.partial.content[1], + ).toMatchObject({ + arguments: {}, + }); + expect(toolDeltas).toMatchObject([ + { contentIndex: 1, delta: '{"city":', partial: { content: [{}, { arguments: {} }] } }, + { + contentIndex: 1, + delta: '"Paris"}', + partial: { content: [{}, { arguments: { city: "Paris" } }] }, + }, + { + contentIndex: 2, + delta: '{"day":"today"}', + partial: { content: [{}, {}, { arguments: { day: "today" } }] }, + }, + ]); + const toolEnds = events.filter((event) => event.type === "toolcall_end"); + expect(toolEnds).toMatchObject([ + { contentIndex: 1, toolCall: { name: "weather", arguments: { city: "Paris" } } }, + { contentIndex: 2, toolCall: { name: "calendar", arguments: { day: "today" } } }, + ]); + const done = events.at(-1); + expect(done).toMatchObject({ + type: "done", + reason: "toolUse", + message: { + content: [ + { type: "text", text: "Checking both." }, + { type: "toolCall", name: "weather", arguments: { city: "Paris" } }, + { type: "toolCall", name: "calendar", arguments: { day: "today" } }, + ], + }, + }); + if (done?.type === "done") { + expect(toolEnds.map((event) => event.toolCall.id)).toEqual( + done.message.content + .filter((content) => content.type === "toolCall") + .map((content) => content.id), + ); + } + }); + + it("never completes or executes an interrupted native call at the token limit", async () => { + mocks.generateResponse.mockImplementationOnce(async (_history, options) => { + options.onFunctionCallParamsChunk({ + callIndex: 0, + functionName: "weather", + paramsChunk: '{"city":', + done: false, + }); + return { + response: "", + functionCalls: undefined, + metadata: { stopReason: "maxTokens" }, + }; + }); + + const events = await collectEvents( + await createLlamaCppStreamFn({})(model, { + messages: [{ role: "user", content: "Weather?", timestamp: 1 }], + tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }], + }), + ); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "done", + ]); + expect(events.at(-1)).toMatchObject({ + type: "done", + reason: "length", + message: { content: [], stopReason: "length" }, + }); + }); + + it("never completes a native call when its final argument reaches the token limit", async () => { + mocks.generateResponse.mockImplementationOnce(async (_history, options) => { + options.onFunctionCallParamsChunk({ + callIndex: 0, + functionName: "weather", + paramsChunk: '{"city":"Paris"}', + done: true, + }); + return { + response: "", + functionCalls: undefined, + metadata: { stopReason: "maxTokens" }, + }; + }); + + const events = await collectEvents( + await createLlamaCppStreamFn({})(model, { + messages: [{ role: "user", content: "Weather?", timestamp: 1 }], + tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }], + }), + ); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "done", + ]); + expect(events.at(-1)).toMatchObject({ + type: "done", + reason: "length", + message: { content: [], stopReason: "length" }, + }); + }); + + it("never lets completed native calls override the authoritative token-limit terminal", async () => { + mocks.generateResponse.mockImplementationOnce(async (_history, options) => { + options.onFunctionCallParamsChunk({ + callIndex: 0, + functionName: "weather", + paramsChunk: '{"city":"Paris"}', + done: true, + }); + options.onFunctionCallParamsChunk({ + callIndex: 1, + functionName: "calendar", + paramsChunk: '{"day":', + done: false, + }); + return { + response: "", + functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }], + metadata: { stopReason: "maxTokens" }, + }; + }); + + const events = await collectEvents( + await createLlamaCppStreamFn({})(model, { + messages: [{ role: "user", content: "Check both", timestamp: 1 }], + tools: [ + { name: "weather", description: "Weather", parameters: { type: "object" } }, + { name: "calendar", description: "Calendar", parameters: { type: "object" } }, + ], + }), + ); + + expect(events.filter((event) => event.type === "toolcall_end")).toHaveLength(0); + expect(events.at(-1)).toMatchObject({ + type: "done", + reason: "length", + message: { + stopReason: "length", + content: [], + }, + }); + }); + it.each([ { format: "Harmony", @@ -521,6 +944,47 @@ describe("llama.cpp inference provider", () => { expect(mocks.llama.loadModel).toHaveBeenCalledTimes(1); }); + it.each([ + { + scenario: "a smaller advertised model window", + model: { ...model, contextWindow: 4096, contextTokens: undefined }, + expectedContextSize: { max: 4096 }, + expectedGpuFit: 4096, + }, + { + scenario: "the safe default below a larger advertised window", + model: { ...model, contextWindow: 32_768, contextTokens: undefined }, + expectedContextSize: { max: 8192 }, + expectedGpuFit: 8192, + }, + { + scenario: "an explicit practical runtime cap", + model: { ...model, contextWindow: 8192, contextTokens: 3072 }, + expectedContextSize: { max: 3072 }, + expectedGpuFit: 3072, + }, + { + scenario: "an explicitly configured native context size", + model: { ...model, contextTokens: 4096, params: { ...model.params, contextSize: 2048 } }, + expectedContextSize: 2048, + expectedGpuFit: 2048, + }, + ])("bounds native context and GPU allocation by $scenario", async (scenario) => { + await collectEvents( + await createLlamaCppStreamFn({})(scenario.model, { + messages: [{ role: "user", content: "Hi", timestamp: 1 }], + }), + ); + expect(mocks.model.createContext).toHaveBeenCalledWith( + expect.objectContaining({ contextSize: scenario.expectedContextSize }), + ); + expect(mocks.llama.loadModel).toHaveBeenCalledWith( + expect.objectContaining({ + gpuLayers: { fitContext: { contextSize: scenario.expectedGpuFit } }, + }), + ); + }); + it("expands home-relative local model paths before resolving the file", async () => { const stream = await createLlamaCppStreamFn({})( { ...model, params: { modelPath: "~/Models/test.gguf" } }, diff --git a/extensions/llama-cpp/src/inference-provider.ts b/extensions/llama-cpp/src/inference-provider.ts index 0de0499bf706..a6d04d927831 100644 --- a/extensions/llama-cpp/src/inference-provider.ts +++ b/extensions/llama-cpp/src/inference-provider.ts @@ -5,6 +5,8 @@ import type { Llama, LlamaContext, LlamaContextSequence, + LlamaChatResponseChunk, + LlamaChatResponseFunctionCallParamsChunk, LlamaModel, } from "node-llama-cpp"; import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; @@ -15,7 +17,7 @@ import type { ToolCall, Usage, } from "openclaw/plugin-sdk/llm"; -import { createAssistantMessageEventStream } from "openclaw/plugin-sdk/llm"; +import { createAssistantMessageEventStream, parseStreamingJson } from "openclaw/plugin-sdk/llm"; import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; import { createPlainTextToolCallCompatWrapper } from "openclaw/plugin-sdk/provider-stream-shared"; import { @@ -220,10 +222,16 @@ function resolveContextSize( if (typeof configured === "number") { return configured; } + const advertisedCap = + typeof model.contextWindow === "number" && model.contextWindow > 0 + ? Math.floor(model.contextWindow) + : DEFAULT_LLAMA_CPP_CONTEXT_SIZE; + // Advertised capacity is a ceiling, not permission to silently exceed the + // established local-memory default; explicit runtime caps can still opt in. const modelCap = typeof model.contextTokens === "number" && model.contextTokens > 0 ? Math.floor(model.contextTokens) - : DEFAULT_LLAMA_CPP_CONTEXT_SIZE; + : Math.min(advertisedCap, DEFAULT_LLAMA_CPP_CONTEXT_SIZE); return { max: modelCap }; } @@ -297,6 +305,7 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC return createPlainTextToolCallCompatWrapper((model, context, options) => { const stream = createAssistantMessageEventStream(); let streamedText = ""; + const streamedContent: AssistantMessage["content"] = []; let generationAborted = false; let started = false; let ended = false; @@ -344,24 +353,148 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC }); const before = sequence.tokenMeter.getState(); const functions = mapToolsToLlamaFunctions(context); - let textStarted = false; + const streamedToolCalls = new Map< + number, + { toolCall: ToolCall; contentIndex: number; partialArgs: string } + >(); + let streamStarted = false; + let activeThinking: { contentIndex: number; thinking: string } | undefined; + let activeText: { contentIndex: number; text: string } | undefined; const partial = () => buildMessage({ model, - content: streamedText ? [{ type: "text", text: streamedText }] : [], + content: [...streamedContent], stopReason: "stop", }); + const ensureStreamStarted = () => { + if (streamStarted) { + return; + } + streamStarted = true; + stream.push({ type: "start", partial: partial() }); + }; + const closeThinkingBlock = () => { + if (!activeThinking) { + return; + } + const thinking = activeThinking; + activeThinking = undefined; + stream.push({ + type: "thinking_end", + contentIndex: thinking.contentIndex, + content: thinking.thinking, + partial: partial(), + }); + }; + const closeTextBlock = () => { + if (!activeText) { + return; + } + const text = activeText; + activeText = undefined; + stream.push({ + type: "text_end", + contentIndex: text.contentIndex, + content: text.text, + partial: partial(), + }); + }; + const appendThinkingChunk = (chunk: LlamaChatResponseChunk) => { + if (chunk.type !== "segment" || chunk.segmentType !== "thought") { + return; + } + if (chunk.text) { + closeTextBlock(); + if (!activeThinking) { + ensureStreamStarted(); + activeThinking = { contentIndex: streamedContent.length, thinking: "" }; + streamedContent.push({ type: "thinking", thinking: "" }); + stream.push({ + type: "thinking_start", + contentIndex: activeThinking.contentIndex, + partial: partial(), + }); + } + activeThinking.thinking += chunk.text; + streamedContent[activeThinking.contentIndex] = { + type: "thinking", + thinking: activeThinking.thinking, + }; + stream.push({ + type: "thinking_delta", + contentIndex: activeThinking.contentIndex, + delta: chunk.text, + partial: partial(), + }); + } + if (chunk.segmentEndTime) { + closeThinkingBlock(); + } + }; const appendTextDelta = (delta: string) => { if (!delta) { return; } - if (!textStarted) { - textStarted = true; - stream.push({ type: "start", partial: partial() }); - stream.push({ type: "text_start", contentIndex: 0, partial: partial() }); + closeThinkingBlock(); + if (!activeText) { + ensureStreamStarted(); + activeText = { contentIndex: streamedContent.length, text: "" }; + streamedContent.push({ type: "text", text: "" }); + stream.push({ + type: "text_start", + contentIndex: activeText.contentIndex, + partial: partial(), + }); } streamedText += delta; - stream.push({ type: "text_delta", contentIndex: 0, delta }); + activeText.text += delta; + streamedContent[activeText.contentIndex] = { type: "text", text: activeText.text }; + stream.push({ + type: "text_delta", + contentIndex: activeText.contentIndex, + delta, + }); + }; + const appendFunctionCallParamsChunk = (chunk: LlamaChatResponseFunctionCallParamsChunk) => { + closeThinkingBlock(); + closeTextBlock(); + let state = streamedToolCalls.get(chunk.callIndex); + if (!state) { + ensureStreamStarted(); + state = { + toolCall: { + type: "toolCall", + id: `llama_cpp_call_${randomUUID()}`, + name: chunk.functionName, + arguments: {}, + }, + contentIndex: streamedContent.length, + partialArgs: "", + }; + streamedToolCalls.set(chunk.callIndex, state); + streamedContent.push(state.toolCall); + stream.push({ + type: "toolcall_start", + contentIndex: state.contentIndex, + partial: partial(), + }); + } + if (chunk.paramsChunk) { + state.partialArgs += chunk.paramsChunk; + // Replace the block so already queued partial snapshots retain the + // exact argument state they exposed before this streamed delta. + state.toolCall = { + ...state.toolCall, + arguments: parseStreamingJson(state.partialArgs), + }; + streamedContent[state.contentIndex] = state.toolCall; + stream.push({ + type: "toolcall_delta", + contentIndex: state.contentIndex, + delta: chunk.paramsChunk, + partial: partial(), + }); + } }; try { // node-llama-cpp makes grammar and functions mutually exclusive. Tool @@ -379,8 +512,13 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC temperature: options?.temperature, customStopTriggers: options?.stop, onTextChunk: appendTextDelta, + ...(model.reasoning ? { onResponseChunk: appendThinkingChunk } : {}), ...(functions - ? { functions, documentFunctionParams: true as const } + ? { + functions, + documentFunctionParams: true as const, + onFunctionCallParamsChunk: appendFunctionCallParamsChunk, + } : grammar ? { grammar } : {}), @@ -397,29 +535,52 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC if (!streamedText && result.response) { appendTextDelta(result.response); } - const content: AssistantMessage["content"] = streamedText - ? [{ type: "text", text: streamedText }] - : []; - if (textStarted) { + closeThinkingBlock(); + closeTextBlock(); + // A max-token result can contain previously completed calls while a + // later call was truncated. Its terminal owns the entire generation. + const confirmedCalls = + result.metadata.stopReason === "maxTokens" ? [] : (result.functionCalls ?? []); + const toolCalls: ToolCall[] = confirmedCalls.map((call, callIndex) => { + let state = streamedToolCalls.get(callIndex); + const argumentsObject = normalizeArguments(call.params); + if (!state) { + appendFunctionCallParamsChunk({ + callIndex, + functionName: call.functionName, + paramsChunk: JSON.stringify(argumentsObject), + done: true, + }); + state = streamedToolCalls.get(callIndex); + } + if (!state) { + throw new Error("llama.cpp native tool call stream state is missing"); + } + state.toolCall = { + ...state.toolCall, + name: call.functionName, + arguments: argumentsObject, + }; + streamedContent[state.contentIndex] = state.toolCall; + // The dependency reports its final argument chunk before checking the + // token budget; only this authoritative result can complete a call. stream.push({ - type: "text_end", - contentIndex: 0, - content: streamedText, + type: "toolcall_end", + contentIndex: state.contentIndex, + toolCall: state.toolCall, partial: partial(), }); - } - const toolCalls: ToolCall[] = (result.functionCalls ?? []).map((call) => ({ - type: "toolCall", - id: `llama_cpp_call_${randomUUID()}`, - name: call.functionName, - arguments: normalizeArguments(call.params), - })); - content.push(...toolCalls); + return state.toolCall; + }); + const confirmedToolCallIds = new Set(toolCalls.map((toolCall) => toolCall.id)); + const content = streamedContent.filter( + (block) => block.type !== "toolCall" || confirmedToolCallIds.has(block.id), + ); const reason: Extract = - toolCalls.length > 0 - ? "toolUse" - : result.metadata.stopReason === "maxTokens" - ? "length" + result.metadata.stopReason === "maxTokens" + ? "length" + : toolCalls.length > 0 + ? "toolUse" : "stop"; const message = buildMessage({ model, @@ -440,7 +601,7 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC reason, error: buildMessage({ model, - content: streamedText ? [{ type: "text", text: streamedText }] : [], + content: streamedContent.filter((block) => block.type !== "toolCall"), stopReason: reason, errorMessage, }),