mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 05:51:39 +00:00
fix(llama-cpp): preserve native tool, context, and reasoning lifecycles (#116903)
* fix(llama-cpp): stream native tool call lifecycle * fix(llama-cpp): normalize native response lifecycle and limits --------- Co-authored-by: Peter Steinberger <steipete@macos.shared>
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
0d209991b6
commit
101bcaf011
@@ -235,6 +235,194 @@ describe("llama.cpp inference provider", () => {
|
||||
maxTokens: 2048,
|
||||
customStopTriggers: ["END"],
|
||||
});
|
||||
expect(mocks.generateResponse.mock.calls[0]?.[1]).not.toHaveProperty("onResponseChunk");
|
||||
});
|
||||
|
||||
it("streams reasoning and text before a result-only native tool call", async () => {
|
||||
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
|
||||
options.onResponseChunk({
|
||||
type: "segment",
|
||||
segmentType: "thought",
|
||||
text: "First ",
|
||||
tokens: [1],
|
||||
});
|
||||
options.onResponseChunk({
|
||||
type: "segment",
|
||||
segmentType: "thought",
|
||||
text: "reason.",
|
||||
tokens: [2],
|
||||
});
|
||||
options.onTextChunk("Answer.");
|
||||
return {
|
||||
response: "Answer.",
|
||||
functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }],
|
||||
metadata: { stopReason: "functionCalls" },
|
||||
};
|
||||
});
|
||||
|
||||
const events = await collectEvents(
|
||||
await createLlamaCppStreamFn({})(
|
||||
{ ...model, reasoning: true },
|
||||
{
|
||||
messages: [{ role: "user", content: "Why?", timestamp: 1 }],
|
||||
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"start",
|
||||
"thinking_start",
|
||||
"thinking_delta",
|
||||
"thinking_delta",
|
||||
"thinking_end",
|
||||
"text_start",
|
||||
"text_delta",
|
||||
"text_end",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "done",
|
||||
message: {
|
||||
content: [
|
||||
{ type: "thinking", thinking: "First reason." },
|
||||
{ type: "text", text: "Answer." },
|
||||
{ type: "toolCall", name: "weather", arguments: { city: "Paris" } },
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(events.find((event) => event.type === "thinking_delta")).toMatchObject({
|
||||
contentIndex: 0,
|
||||
partial: { content: [{ type: "thinking", thinking: "First " }] },
|
||||
});
|
||||
expect(events.find((event) => event.type === "toolcall_start")).toMatchObject({
|
||||
contentIndex: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("closes native reasoning before streaming a completed tool call", async () => {
|
||||
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
|
||||
options.onResponseChunk({
|
||||
type: "segment",
|
||||
segmentType: "thought",
|
||||
text: "Need current weather.",
|
||||
tokens: [1],
|
||||
});
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 0,
|
||||
functionName: "weather",
|
||||
paramsChunk: '{"city":"Paris"}',
|
||||
done: true,
|
||||
});
|
||||
return {
|
||||
response: "",
|
||||
functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }],
|
||||
metadata: { stopReason: "functionCalls" },
|
||||
};
|
||||
});
|
||||
|
||||
const events = await collectEvents(
|
||||
await createLlamaCppStreamFn({})(
|
||||
{ ...model, reasoning: true },
|
||||
{
|
||||
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
|
||||
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"start",
|
||||
"thinking_start",
|
||||
"thinking_delta",
|
||||
"thinking_end",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
expect(events.find((event) => event.type === "toolcall_start")).toMatchObject({
|
||||
contentIndex: 1,
|
||||
});
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "done",
|
||||
reason: "toolUse",
|
||||
message: {
|
||||
content: [
|
||||
{ type: "thinking", thinking: "Need current weather." },
|
||||
{ type: "toolCall", name: "weather", arguments: { city: "Paris" } },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("opens a new indexed block for every thought segment after visible text", async () => {
|
||||
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
|
||||
options.onResponseChunk({
|
||||
type: "segment",
|
||||
segmentType: "thought",
|
||||
text: "First thought.",
|
||||
tokens: [1],
|
||||
segmentEndTime: new Date(1),
|
||||
});
|
||||
options.onTextChunk("First answer.");
|
||||
options.onResponseChunk({
|
||||
type: "segment",
|
||||
segmentType: "thought",
|
||||
text: "Second thought.",
|
||||
tokens: [2],
|
||||
segmentEndTime: new Date(2),
|
||||
});
|
||||
options.onTextChunk("Second answer.");
|
||||
return {
|
||||
response: "First answer.Second answer.",
|
||||
functionCalls: undefined,
|
||||
metadata: { stopReason: "eogToken" },
|
||||
};
|
||||
});
|
||||
|
||||
const events = await collectEvents(
|
||||
await createLlamaCppStreamFn({})(
|
||||
{ ...model, reasoning: true },
|
||||
{ messages: [{ role: "user", content: "Reason twice", timestamp: 1 }] },
|
||||
),
|
||||
);
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"start",
|
||||
"thinking_start",
|
||||
"thinking_delta",
|
||||
"thinking_end",
|
||||
"text_start",
|
||||
"text_delta",
|
||||
"text_end",
|
||||
"thinking_start",
|
||||
"thinking_delta",
|
||||
"thinking_end",
|
||||
"text_start",
|
||||
"text_delta",
|
||||
"text_end",
|
||||
"done",
|
||||
]);
|
||||
expect(
|
||||
events
|
||||
.filter((event) => event.type === "thinking_start" || event.type === "text_start")
|
||||
.map((event) => event.contentIndex),
|
||||
).toEqual([0, 1, 2, 3]);
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "done",
|
||||
message: {
|
||||
content: [
|
||||
{ type: "thinking", thinking: "First thought." },
|
||||
{ type: "text", text: "First answer." },
|
||||
{ type: "thinking", thinking: "Second thought." },
|
||||
{ type: "text", text: "Second answer." },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("builds a JSON Schema grammar for tool-free responseFormat requests", async () => {
|
||||
@@ -327,7 +515,7 @@ describe("llama.cpp inference provider", () => {
|
||||
expect(mocks.generateResponse.mock.calls[0]?.[1]).not.toHaveProperty("grammar");
|
||||
});
|
||||
|
||||
it("emits native function calls in the final assistant message", async () => {
|
||||
it("streams the complete lifecycle for native function calls", async () => {
|
||||
mocks.generateResponse.mockResolvedValueOnce({
|
||||
response: "",
|
||||
functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }],
|
||||
@@ -346,7 +534,31 @@ describe("llama.cpp inference provider", () => {
|
||||
|
||||
const events = await collectEvents(stream);
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual(["done"]);
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
const toolCallStart = events[1];
|
||||
const toolCallDelta = events[2];
|
||||
const toolCallEnd = events[3];
|
||||
expect(toolCallStart).toMatchObject({
|
||||
type: "toolcall_start",
|
||||
contentIndex: 0,
|
||||
partial: { content: [{ type: "toolCall", name: "weather", arguments: {} }] },
|
||||
});
|
||||
expect(toolCallDelta).toMatchObject({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: 0,
|
||||
delta: '{"city":"Paris"}',
|
||||
});
|
||||
expect(toolCallEnd).toMatchObject({
|
||||
type: "toolcall_end",
|
||||
contentIndex: 0,
|
||||
toolCall: { name: "weather", arguments: { city: "Paris" } },
|
||||
});
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "done",
|
||||
reason: "toolUse",
|
||||
@@ -364,6 +576,217 @@ describe("llama.cpp inference provider", () => {
|
||||
expect(mocks.llama.createGrammarForJsonSchema).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("streams split native arguments with stable ids after mixed text", async () => {
|
||||
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
|
||||
options.onTextChunk("Checking both.");
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 0,
|
||||
functionName: "weather",
|
||||
paramsChunk: '{"city":',
|
||||
done: false,
|
||||
});
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 0,
|
||||
functionName: "weather",
|
||||
paramsChunk: '"Paris"}',
|
||||
done: true,
|
||||
});
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 1,
|
||||
functionName: "calendar",
|
||||
paramsChunk: '{"day":"today"}',
|
||||
done: true,
|
||||
});
|
||||
return {
|
||||
response: "Checking both.",
|
||||
functionCalls: [
|
||||
{ functionName: "weather", params: { city: "Paris" }, raw: [] },
|
||||
{ functionName: "calendar", params: { day: "today" }, raw: [] },
|
||||
],
|
||||
metadata: { stopReason: "functionCalls" },
|
||||
};
|
||||
});
|
||||
|
||||
const stream = await createLlamaCppStreamFn({})(model, {
|
||||
messages: [{ role: "user", content: "Check both", timestamp: 1 }],
|
||||
tools: [
|
||||
{ name: "weather", description: "Weather", parameters: { type: "object" } },
|
||||
{ name: "calendar", description: "Calendar", parameters: { type: "object" } },
|
||||
],
|
||||
});
|
||||
const events = await collectEvents(stream);
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"start",
|
||||
"text_start",
|
||||
"text_delta",
|
||||
"text_end",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_delta",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
const toolDeltas = events.filter((event) => event.type === "toolcall_delta");
|
||||
expect(
|
||||
events.find((event) => event.type === "toolcall_start")?.partial.content[1],
|
||||
).toMatchObject({
|
||||
arguments: {},
|
||||
});
|
||||
expect(toolDeltas).toMatchObject([
|
||||
{ contentIndex: 1, delta: '{"city":', partial: { content: [{}, { arguments: {} }] } },
|
||||
{
|
||||
contentIndex: 1,
|
||||
delta: '"Paris"}',
|
||||
partial: { content: [{}, { arguments: { city: "Paris" } }] },
|
||||
},
|
||||
{
|
||||
contentIndex: 2,
|
||||
delta: '{"day":"today"}',
|
||||
partial: { content: [{}, {}, { arguments: { day: "today" } }] },
|
||||
},
|
||||
]);
|
||||
const toolEnds = events.filter((event) => event.type === "toolcall_end");
|
||||
expect(toolEnds).toMatchObject([
|
||||
{ contentIndex: 1, toolCall: { name: "weather", arguments: { city: "Paris" } } },
|
||||
{ contentIndex: 2, toolCall: { name: "calendar", arguments: { day: "today" } } },
|
||||
]);
|
||||
const done = events.at(-1);
|
||||
expect(done).toMatchObject({
|
||||
type: "done",
|
||||
reason: "toolUse",
|
||||
message: {
|
||||
content: [
|
||||
{ type: "text", text: "Checking both." },
|
||||
{ type: "toolCall", name: "weather", arguments: { city: "Paris" } },
|
||||
{ type: "toolCall", name: "calendar", arguments: { day: "today" } },
|
||||
],
|
||||
},
|
||||
});
|
||||
if (done?.type === "done") {
|
||||
expect(toolEnds.map((event) => event.toolCall.id)).toEqual(
|
||||
done.message.content
|
||||
.filter((content) => content.type === "toolCall")
|
||||
.map((content) => content.id),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("never completes or executes an interrupted native call at the token limit", async () => {
|
||||
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 0,
|
||||
functionName: "weather",
|
||||
paramsChunk: '{"city":',
|
||||
done: false,
|
||||
});
|
||||
return {
|
||||
response: "",
|
||||
functionCalls: undefined,
|
||||
metadata: { stopReason: "maxTokens" },
|
||||
};
|
||||
});
|
||||
|
||||
const events = await collectEvents(
|
||||
await createLlamaCppStreamFn({})(model, {
|
||||
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
|
||||
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"done",
|
||||
]);
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "done",
|
||||
reason: "length",
|
||||
message: { content: [], stopReason: "length" },
|
||||
});
|
||||
});
|
||||
|
||||
it("never completes a native call when its final argument reaches the token limit", async () => {
|
||||
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 0,
|
||||
functionName: "weather",
|
||||
paramsChunk: '{"city":"Paris"}',
|
||||
done: true,
|
||||
});
|
||||
return {
|
||||
response: "",
|
||||
functionCalls: undefined,
|
||||
metadata: { stopReason: "maxTokens" },
|
||||
};
|
||||
});
|
||||
|
||||
const events = await collectEvents(
|
||||
await createLlamaCppStreamFn({})(model, {
|
||||
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
|
||||
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"done",
|
||||
]);
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "done",
|
||||
reason: "length",
|
||||
message: { content: [], stopReason: "length" },
|
||||
});
|
||||
});
|
||||
|
||||
it("never lets completed native calls override the authoritative token-limit terminal", async () => {
|
||||
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 0,
|
||||
functionName: "weather",
|
||||
paramsChunk: '{"city":"Paris"}',
|
||||
done: true,
|
||||
});
|
||||
options.onFunctionCallParamsChunk({
|
||||
callIndex: 1,
|
||||
functionName: "calendar",
|
||||
paramsChunk: '{"day":',
|
||||
done: false,
|
||||
});
|
||||
return {
|
||||
response: "",
|
||||
functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }],
|
||||
metadata: { stopReason: "maxTokens" },
|
||||
};
|
||||
});
|
||||
|
||||
const events = await collectEvents(
|
||||
await createLlamaCppStreamFn({})(model, {
|
||||
messages: [{ role: "user", content: "Check both", timestamp: 1 }],
|
||||
tools: [
|
||||
{ name: "weather", description: "Weather", parameters: { type: "object" } },
|
||||
{ name: "calendar", description: "Calendar", parameters: { type: "object" } },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(events.filter((event) => event.type === "toolcall_end")).toHaveLength(0);
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "done",
|
||||
reason: "length",
|
||||
message: {
|
||||
stopReason: "length",
|
||||
content: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
format: "Harmony",
|
||||
@@ -521,6 +944,47 @@ describe("llama.cpp inference provider", () => {
|
||||
expect(mocks.llama.loadModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
scenario: "a smaller advertised model window",
|
||||
model: { ...model, contextWindow: 4096, contextTokens: undefined },
|
||||
expectedContextSize: { max: 4096 },
|
||||
expectedGpuFit: 4096,
|
||||
},
|
||||
{
|
||||
scenario: "the safe default below a larger advertised window",
|
||||
model: { ...model, contextWindow: 32_768, contextTokens: undefined },
|
||||
expectedContextSize: { max: 8192 },
|
||||
expectedGpuFit: 8192,
|
||||
},
|
||||
{
|
||||
scenario: "an explicit practical runtime cap",
|
||||
model: { ...model, contextWindow: 8192, contextTokens: 3072 },
|
||||
expectedContextSize: { max: 3072 },
|
||||
expectedGpuFit: 3072,
|
||||
},
|
||||
{
|
||||
scenario: "an explicitly configured native context size",
|
||||
model: { ...model, contextTokens: 4096, params: { ...model.params, contextSize: 2048 } },
|
||||
expectedContextSize: 2048,
|
||||
expectedGpuFit: 2048,
|
||||
},
|
||||
])("bounds native context and GPU allocation by $scenario", async (scenario) => {
|
||||
await collectEvents(
|
||||
await createLlamaCppStreamFn({})(scenario.model, {
|
||||
messages: [{ role: "user", content: "Hi", timestamp: 1 }],
|
||||
}),
|
||||
);
|
||||
expect(mocks.model.createContext).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ contextSize: scenario.expectedContextSize }),
|
||||
);
|
||||
expect(mocks.llama.loadModel).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
gpuLayers: { fitContext: { contextSize: scenario.expectedGpuFit } },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("expands home-relative local model paths before resolving the file", async () => {
|
||||
const stream = await createLlamaCppStreamFn({})(
|
||||
{ ...model, params: { modelPath: "~/Models/test.gguf" } },
|
||||
|
||||
@@ -5,6 +5,8 @@ import type {
|
||||
Llama,
|
||||
LlamaContext,
|
||||
LlamaContextSequence,
|
||||
LlamaChatResponseChunk,
|
||||
LlamaChatResponseFunctionCallParamsChunk,
|
||||
LlamaModel,
|
||||
} from "node-llama-cpp";
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
@@ -15,7 +17,7 @@ import type {
|
||||
ToolCall,
|
||||
Usage,
|
||||
} from "openclaw/plugin-sdk/llm";
|
||||
import { createAssistantMessageEventStream } from "openclaw/plugin-sdk/llm";
|
||||
import { createAssistantMessageEventStream, parseStreamingJson } from "openclaw/plugin-sdk/llm";
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { createPlainTextToolCallCompatWrapper } from "openclaw/plugin-sdk/provider-stream-shared";
|
||||
import {
|
||||
@@ -220,10 +222,16 @@ function resolveContextSize(
|
||||
if (typeof configured === "number") {
|
||||
return configured;
|
||||
}
|
||||
const advertisedCap =
|
||||
typeof model.contextWindow === "number" && model.contextWindow > 0
|
||||
? Math.floor(model.contextWindow)
|
||||
: DEFAULT_LLAMA_CPP_CONTEXT_SIZE;
|
||||
// Advertised capacity is a ceiling, not permission to silently exceed the
|
||||
// established local-memory default; explicit runtime caps can still opt in.
|
||||
const modelCap =
|
||||
typeof model.contextTokens === "number" && model.contextTokens > 0
|
||||
? Math.floor(model.contextTokens)
|
||||
: DEFAULT_LLAMA_CPP_CONTEXT_SIZE;
|
||||
: Math.min(advertisedCap, DEFAULT_LLAMA_CPP_CONTEXT_SIZE);
|
||||
return { max: modelCap };
|
||||
}
|
||||
|
||||
@@ -297,6 +305,7 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC
|
||||
return createPlainTextToolCallCompatWrapper((model, context, options) => {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
let streamedText = "";
|
||||
const streamedContent: AssistantMessage["content"] = [];
|
||||
let generationAborted = false;
|
||||
let started = false;
|
||||
let ended = false;
|
||||
@@ -344,24 +353,148 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC
|
||||
});
|
||||
const before = sequence.tokenMeter.getState();
|
||||
const functions = mapToolsToLlamaFunctions(context);
|
||||
let textStarted = false;
|
||||
const streamedToolCalls = new Map<
|
||||
number,
|
||||
{ toolCall: ToolCall; contentIndex: number; partialArgs: string }
|
||||
>();
|
||||
let streamStarted = false;
|
||||
let activeThinking: { contentIndex: number; thinking: string } | undefined;
|
||||
let activeText: { contentIndex: number; text: string } | undefined;
|
||||
const partial = () =>
|
||||
buildMessage({
|
||||
model,
|
||||
content: streamedText ? [{ type: "text", text: streamedText }] : [],
|
||||
content: [...streamedContent],
|
||||
stopReason: "stop",
|
||||
});
|
||||
const ensureStreamStarted = () => {
|
||||
if (streamStarted) {
|
||||
return;
|
||||
}
|
||||
streamStarted = true;
|
||||
stream.push({ type: "start", partial: partial() });
|
||||
};
|
||||
const closeThinkingBlock = () => {
|
||||
if (!activeThinking) {
|
||||
return;
|
||||
}
|
||||
const thinking = activeThinking;
|
||||
activeThinking = undefined;
|
||||
stream.push({
|
||||
type: "thinking_end",
|
||||
contentIndex: thinking.contentIndex,
|
||||
content: thinking.thinking,
|
||||
partial: partial(),
|
||||
});
|
||||
};
|
||||
const closeTextBlock = () => {
|
||||
if (!activeText) {
|
||||
return;
|
||||
}
|
||||
const text = activeText;
|
||||
activeText = undefined;
|
||||
stream.push({
|
||||
type: "text_end",
|
||||
contentIndex: text.contentIndex,
|
||||
content: text.text,
|
||||
partial: partial(),
|
||||
});
|
||||
};
|
||||
const appendThinkingChunk = (chunk: LlamaChatResponseChunk) => {
|
||||
if (chunk.type !== "segment" || chunk.segmentType !== "thought") {
|
||||
return;
|
||||
}
|
||||
if (chunk.text) {
|
||||
closeTextBlock();
|
||||
if (!activeThinking) {
|
||||
ensureStreamStarted();
|
||||
activeThinking = { contentIndex: streamedContent.length, thinking: "" };
|
||||
streamedContent.push({ type: "thinking", thinking: "" });
|
||||
stream.push({
|
||||
type: "thinking_start",
|
||||
contentIndex: activeThinking.contentIndex,
|
||||
partial: partial(),
|
||||
});
|
||||
}
|
||||
activeThinking.thinking += chunk.text;
|
||||
streamedContent[activeThinking.contentIndex] = {
|
||||
type: "thinking",
|
||||
thinking: activeThinking.thinking,
|
||||
};
|
||||
stream.push({
|
||||
type: "thinking_delta",
|
||||
contentIndex: activeThinking.contentIndex,
|
||||
delta: chunk.text,
|
||||
partial: partial(),
|
||||
});
|
||||
}
|
||||
if (chunk.segmentEndTime) {
|
||||
closeThinkingBlock();
|
||||
}
|
||||
};
|
||||
const appendTextDelta = (delta: string) => {
|
||||
if (!delta) {
|
||||
return;
|
||||
}
|
||||
if (!textStarted) {
|
||||
textStarted = true;
|
||||
stream.push({ type: "start", partial: partial() });
|
||||
stream.push({ type: "text_start", contentIndex: 0, partial: partial() });
|
||||
closeThinkingBlock();
|
||||
if (!activeText) {
|
||||
ensureStreamStarted();
|
||||
activeText = { contentIndex: streamedContent.length, text: "" };
|
||||
streamedContent.push({ type: "text", text: "" });
|
||||
stream.push({
|
||||
type: "text_start",
|
||||
contentIndex: activeText.contentIndex,
|
||||
partial: partial(),
|
||||
});
|
||||
}
|
||||
streamedText += delta;
|
||||
stream.push({ type: "text_delta", contentIndex: 0, delta });
|
||||
activeText.text += delta;
|
||||
streamedContent[activeText.contentIndex] = { type: "text", text: activeText.text };
|
||||
stream.push({
|
||||
type: "text_delta",
|
||||
contentIndex: activeText.contentIndex,
|
||||
delta,
|
||||
});
|
||||
};
|
||||
const appendFunctionCallParamsChunk = (chunk: LlamaChatResponseFunctionCallParamsChunk) => {
|
||||
closeThinkingBlock();
|
||||
closeTextBlock();
|
||||
let state = streamedToolCalls.get(chunk.callIndex);
|
||||
if (!state) {
|
||||
ensureStreamStarted();
|
||||
state = {
|
||||
toolCall: {
|
||||
type: "toolCall",
|
||||
id: `llama_cpp_call_${randomUUID()}`,
|
||||
name: chunk.functionName,
|
||||
arguments: {},
|
||||
},
|
||||
contentIndex: streamedContent.length,
|
||||
partialArgs: "",
|
||||
};
|
||||
streamedToolCalls.set(chunk.callIndex, state);
|
||||
streamedContent.push(state.toolCall);
|
||||
stream.push({
|
||||
type: "toolcall_start",
|
||||
contentIndex: state.contentIndex,
|
||||
partial: partial(),
|
||||
});
|
||||
}
|
||||
if (chunk.paramsChunk) {
|
||||
state.partialArgs += chunk.paramsChunk;
|
||||
// Replace the block so already queued partial snapshots retain the
|
||||
// exact argument state they exposed before this streamed delta.
|
||||
state.toolCall = {
|
||||
...state.toolCall,
|
||||
arguments: parseStreamingJson(state.partialArgs),
|
||||
};
|
||||
streamedContent[state.contentIndex] = state.toolCall;
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: state.contentIndex,
|
||||
delta: chunk.paramsChunk,
|
||||
partial: partial(),
|
||||
});
|
||||
}
|
||||
};
|
||||
try {
|
||||
// node-llama-cpp makes grammar and functions mutually exclusive. Tool
|
||||
@@ -379,8 +512,13 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC
|
||||
temperature: options?.temperature,
|
||||
customStopTriggers: options?.stop,
|
||||
onTextChunk: appendTextDelta,
|
||||
...(model.reasoning ? { onResponseChunk: appendThinkingChunk } : {}),
|
||||
...(functions
|
||||
? { functions, documentFunctionParams: true as const }
|
||||
? {
|
||||
functions,
|
||||
documentFunctionParams: true as const,
|
||||
onFunctionCallParamsChunk: appendFunctionCallParamsChunk,
|
||||
}
|
||||
: grammar
|
||||
? { grammar }
|
||||
: {}),
|
||||
@@ -397,29 +535,52 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC
|
||||
if (!streamedText && result.response) {
|
||||
appendTextDelta(result.response);
|
||||
}
|
||||
const content: AssistantMessage["content"] = streamedText
|
||||
? [{ type: "text", text: streamedText }]
|
||||
: [];
|
||||
if (textStarted) {
|
||||
closeThinkingBlock();
|
||||
closeTextBlock();
|
||||
// A max-token result can contain previously completed calls while a
|
||||
// later call was truncated. Its terminal owns the entire generation.
|
||||
const confirmedCalls =
|
||||
result.metadata.stopReason === "maxTokens" ? [] : (result.functionCalls ?? []);
|
||||
const toolCalls: ToolCall[] = confirmedCalls.map((call, callIndex) => {
|
||||
let state = streamedToolCalls.get(callIndex);
|
||||
const argumentsObject = normalizeArguments(call.params);
|
||||
if (!state) {
|
||||
appendFunctionCallParamsChunk({
|
||||
callIndex,
|
||||
functionName: call.functionName,
|
||||
paramsChunk: JSON.stringify(argumentsObject),
|
||||
done: true,
|
||||
});
|
||||
state = streamedToolCalls.get(callIndex);
|
||||
}
|
||||
if (!state) {
|
||||
throw new Error("llama.cpp native tool call stream state is missing");
|
||||
}
|
||||
state.toolCall = {
|
||||
...state.toolCall,
|
||||
name: call.functionName,
|
||||
arguments: argumentsObject,
|
||||
};
|
||||
streamedContent[state.contentIndex] = state.toolCall;
|
||||
// The dependency reports its final argument chunk before checking the
|
||||
// token budget; only this authoritative result can complete a call.
|
||||
stream.push({
|
||||
type: "text_end",
|
||||
contentIndex: 0,
|
||||
content: streamedText,
|
||||
type: "toolcall_end",
|
||||
contentIndex: state.contentIndex,
|
||||
toolCall: state.toolCall,
|
||||
partial: partial(),
|
||||
});
|
||||
}
|
||||
const toolCalls: ToolCall[] = (result.functionCalls ?? []).map((call) => ({
|
||||
type: "toolCall",
|
||||
id: `llama_cpp_call_${randomUUID()}`,
|
||||
name: call.functionName,
|
||||
arguments: normalizeArguments(call.params),
|
||||
}));
|
||||
content.push(...toolCalls);
|
||||
return state.toolCall;
|
||||
});
|
||||
const confirmedToolCallIds = new Set(toolCalls.map((toolCall) => toolCall.id));
|
||||
const content = streamedContent.filter(
|
||||
(block) => block.type !== "toolCall" || confirmedToolCallIds.has(block.id),
|
||||
);
|
||||
const reason: Extract<StopReason, "stop" | "length" | "toolUse"> =
|
||||
toolCalls.length > 0
|
||||
? "toolUse"
|
||||
: result.metadata.stopReason === "maxTokens"
|
||||
? "length"
|
||||
result.metadata.stopReason === "maxTokens"
|
||||
? "length"
|
||||
: toolCalls.length > 0
|
||||
? "toolUse"
|
||||
: "stop";
|
||||
const message = buildMessage({
|
||||
model,
|
||||
@@ -440,7 +601,7 @@ export function createLlamaCppStreamFn(params: { providerConfig?: ModelProviderC
|
||||
reason,
|
||||
error: buildMessage({
|
||||
model,
|
||||
content: streamedText ? [{ type: "text", text: streamedText }] : [],
|
||||
content: streamedContent.filter((block) => block.type !== "toolCall"),
|
||||
stopReason: reason,
|
||||
errorMessage,
|
||||
}),
|
||||
|
||||
Reference in New Issue
Block a user