From a079d79f20391b1300d18a6ca3af042e481bee2d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 30 Jul 2026 13:31:46 -0700 Subject: [PATCH] fix(gateway): honor session history Accept negotiation (#116535) * fix(gateway): parse session history Accept ranges * fix(gateway): enforce HTTP media parameter grammar * fix(gateway): honor Accept parameter ordering --- src/gateway/http-common.ts | 4 +- src/gateway/http-media-range.ts | 199 ++++++++++++++++++++++ src/gateway/sessions-history-http.test.ts | 198 +++++++++++++++++++++ src/gateway/sessions-history-http.ts | 10 +- 4 files changed, 404 insertions(+), 7 deletions(-) create mode 100644 src/gateway/http-media-range.ts diff --git a/src/gateway/http-common.ts b/src/gateway/http-common.ts index 9f5e22acdf65..265bb53a4c51 100644 --- a/src/gateway/http-common.ts +++ b/src/gateway/http-common.ts @@ -142,9 +142,11 @@ export function writeDone(res: ServerResponse) { res.write("data: [DONE]\n\n"); } +export const SSE_CONTENT_TYPE = "text/event-stream; charset=utf-8"; + export function setSseHeaders(res: ServerResponse) { res.statusCode = 200; - res.setHeader("Content-Type", "text/event-stream; charset=utf-8"); + res.setHeader("Content-Type", SSE_CONTENT_TYPE); res.setHeader("Cache-Control", "no-cache"); res.setHeader("Connection", "keep-alive"); res.flushHeaders?.(); diff --git a/src/gateway/http-media-range.ts b/src/gateway/http-media-range.ts new file mode 100644 index 000000000000..460b5deee6b5 --- /dev/null +++ b/src/gateway/http-media-range.ts @@ -0,0 +1,199 @@ +// Pure helpers for HTTP Accept media-range parsing. + +const HTTP_TOKEN_PATTERN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/u; +const HTTP_QVALUE_PATTERN = /^(?:0(?:\.\d{0,3})?|1(?:\.0{0,3})?)$/u; +const HTTP_OPTIONAL_WHITESPACE_PATTERN = /^[\t ]+|[\t ]+$/gu; + +function trimHttpOptionalWhitespace(value: string): string { + return value.replace(HTTP_OPTIONAL_WHITESPACE_PATTERN, ""); +} + +function splitOutsideQuotedStrings(value: string, delimiter: string): string[] | null { + const parts: string[] = []; + let start = 0; + let quoted = false; + let escaped = false; + + for (let index = 0; index < value.length; index += 1) { + const character = value[index]; + if (quoted) { + if (escaped) { + escaped = false; + continue; + } + if (character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + quoted = false; + } + continue; + } + if (character === '"') { + quoted = true; + continue; + } + if (character === delimiter) { + parts.push(value.slice(start, index)); + start = index + 1; + } + } + + if (quoted || escaped) { + return null; + } + parts.push(value.slice(start)); + return parts; +} + +function parseParameterValue(value: string): string | null { + if (HTTP_TOKEN_PATTERN.test(value)) { + return value; + } + if (value.length < 2 || value[0] !== '"' || value.at(-1) !== '"') { + return null; + } + let parsed = ""; + let escaped = false; + for (let index = 1; index < value.length - 1; index += 1) { + const character = value[index]; + if (escaped) { + parsed += character; + escaped = false; + continue; + } + if (character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + return null; + } + parsed += character; + } + return escaped ? null : parsed; +} + +function normalizeParameterValue(name: string, value: string): string { + return name === "charset" ? value.toLowerCase() : value; +} + +type ParsedMediaType = { + type: string; + subtype: string; + parameters: Map; + quality: number; +}; + +function parseMediaType(value: string, allowQuality: boolean): ParsedMediaType | null { + const segments = splitOutsideQuotedStrings(value, ";"); + if (!segments) { + return null; + } + const mediaType = trimHttpOptionalWhitespace(segments.shift() ?? "").toLowerCase(); + const [type, subtype, ...extra] = mediaType.split("/"); + if ( + extra.length > 0 || + !type || + !subtype || + !HTTP_TOKEN_PATTERN.test(type) || + !HTTP_TOKEN_PATTERN.test(subtype) + ) { + return null; + } + + let quality = 1; + let qualitySeen = false; + const parameters = new Map(); + for (const rawParameter of segments) { + const parameter = trimHttpOptionalWhitespace(rawParameter); + // RFC 9110 permits a semicolon whose optional parameter is omitted. + if (!parameter) { + continue; + } + const separator = parameter.indexOf("="); + const name = (separator < 0 ? parameter : parameter.slice(0, separator)).toLowerCase(); + if (!HTTP_TOKEN_PATTERN.test(name)) { + return null; + } + if (separator <= 0) { + return null; + } + // Parameter grammar has no whitespace around `=`; accepting it can route + // a malformed negotiation request into a persistent SSE response. + const rawValue = parameter.slice(separator + 1); + if (!rawValue) { + return null; + } + if (name === "q") { + // RFC 9110 recognizes q as the weight regardless of parameter order; + // every other parameter still participates in representation matching. + if (!allowQuality || qualitySeen || !HTTP_QVALUE_PATTERN.test(rawValue)) { + return null; + } + qualitySeen = true; + quality = Number(rawValue); + continue; + } + const parameterValue = parseParameterValue(rawValue); + if (parameterValue === null || parameters.has(name)) { + return null; + } + parameters.set(name, normalizeParameterValue(name, parameterValue)); + } + return { type, subtype, parameters, quality }; +} + +function matchesRepresentation(range: ParsedMediaType, representation: ParsedMediaType): boolean { + if (range.type !== representation.type || range.subtype !== representation.subtype) { + return false; + } + for (const [name, value] of range.parameters) { + if (representation.parameters.get(name) !== value) { + return false; + } + } + return true; +} + +/** + * Checks for an explicit, positive-quality media range in an Accept field. + * Wildcards intentionally do not opt callers into long-lived streaming responses. + */ +export function hasExplicitAcceptableMediaRange( + accept: string | undefined, + expectedRepresentation: string, +): boolean { + if (!accept) { + return false; + } + const representation = parseMediaType(expectedRepresentation, false); + if (!representation) { + return false; + } + + const ranges = splitOutsideQuotedStrings(accept, ","); + if (!ranges) { + return false; + } + let bestSpecificity = -1; + let bestExplicitQuality = 0; + for (const range of ranges) { + if (!trimHttpOptionalWhitespace(range)) { + continue; + } + const parsedRange = parseMediaType(range, true); + if (!parsedRange || !matchesRepresentation(parsedRange, representation)) { + continue; + } + const specificity = parsedRange.parameters.size; + if (specificity > bestSpecificity) { + bestSpecificity = specificity; + bestExplicitQuality = parsedRange.quality; + } else if (specificity === bestSpecificity) { + bestExplicitQuality = Math.max(bestExplicitQuality, parsedRange.quality); + } + } + return bestSpecificity >= 0 && bestExplicitQuality > 0; +} diff --git a/src/gateway/sessions-history-http.test.ts b/src/gateway/sessions-history-http.test.ts index 94bc22889b0f..808aa157e239 100644 --- a/src/gateway/sessions-history-http.test.ts +++ b/src/gateway/sessions-history-http.test.ts @@ -16,6 +16,8 @@ import { emitSessionTranscriptUpdate } from "../sessions/transcript-events.js"; import { OPENCLAW_TRANSCRIPT_ARTIFACT_API } from "../shared/transcript-only-openclaw-assistant.js"; import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; import { runOpenClawAgentWriteTransaction } from "../state/openclaw-agent-db.js"; +import { SSE_CONTENT_TYPE } from "./http-common.js"; +import { hasExplicitAcceptableMediaRange } from "./http-media-range.js"; import { SessionHistorySseState } from "./session-history-state.js"; import { testState } from "./test-helpers.runtime-state.js"; import { @@ -450,7 +452,203 @@ async function openBoundedHistoryStreamWithSecondMessage( return stream; } +describe("session history Accept parsing", () => { + test.each([ + { accept: undefined, expected: false, name: "missing field" }, + { accept: "", expected: false, name: "empty field" }, + { accept: "application/json", expected: false, name: "JSON only" }, + { accept: "text/event-stream", expected: true, name: "exact media type" }, + { accept: "TEXT/EVENT-STREAM", expected: true, name: "case-insensitive media type" }, + { accept: " text/event-stream ", expected: true, name: "optional whitespace" }, + { accept: "text/event-stream;", expected: true, name: "omitted trailing parameter" }, + { + accept: "text/event-stream; ; q=0.5;", + expected: true, + name: "omitted parameter slots", + }, + { + accept: "text/event-stream; charset=utf-8", + expected: true, + name: "media parameter", + }, + { + accept: 'text/event-stream; note="quoted,comma;semicolon\\\"quote"; q=0.5', + expected: false, + name: "quoted and escaped unmatched parameter delimiters", + }, + { + accept: 'text/event-stream; profile="quoted,comma;semicolon\\\"quote"; q=0.5', + expected: true, + name: "quoted and escaped matching parameter delimiters", + representation: 'text/event-stream; profile="quoted,comma;semicolon\\\"quote"', + }, + { + accept: 'text/event-stream; profile="https://example.test/profile"', + expected: false, + name: "case-sensitive parameter mismatch", + representation: 'text/event-stream; profile="https://example.test/Profile"', + }, + { + accept: "text/event-stream; charset=UTF-8", + expected: true, + name: "case-insensitive charset parameter", + }, + { accept: "text/event-stream;q=0.001", expected: true, name: "minimum positive qvalue" }, + { accept: "text/event-stream;Q=1.000", expected: true, name: "maximum qvalue" }, + { + accept: "application/json, text/event-stream;q=0.5", + expected: true, + name: "explicit media range in a list", + }, + { + accept: "text/event-stream;q=0, text/event-stream;q=0.5", + expected: true, + name: "duplicate exact ranges with a positive quality", + }, + { + accept: "text/event-stream;q=1, text/event-stream;charset=utf-8;q=0", + expected: false, + name: "more-specific matching parameter rejection", + }, + { + accept: "text/event-stream;q=0, text/event-stream;charset=utf-8;q=0.5", + expected: true, + name: "more-specific matching parameter acceptance", + }, + { + accept: "text/event-stream;q=0.5;charset=utf-8", + expected: true, + name: "matching media parameter after q", + }, + { + accept: "text/event-stream;q=1;charset=utf-16", + expected: false, + name: "mismatched media parameter after q", + }, + { + accept: "text/event-stream; charset=utf-16", + expected: false, + name: "mismatched representation parameter", + }, + { accept: "text/event-streaming", expected: false, name: "lookalike subtype" }, + { accept: "text/event-streamx", expected: false, name: "suffixed subtype" }, + { + accept: 'application/json; note="text/event-stream"', + expected: false, + name: "quoted parameter decoy", + }, + { accept: "text/*", expected: false, name: "type wildcard" }, + { accept: "*/*", expected: false, name: "all wildcard" }, + { accept: "text/event-stream;q=0", expected: false, name: "zero qvalue" }, + { accept: "text/event-stream;q=0.000", expected: false, name: "zero decimal qvalue" }, + { + accept: "text/event-stream;q=0, */*;q=1", + expected: false, + name: "explicit rejection overriding wildcard", + }, + { accept: "text/event-stream;q=.5", expected: false, name: "missing leading zero" }, + { accept: "text/event-stream;q =0.5", expected: false, name: "whitespace before equals" }, + { accept: "text/event-stream;q= 0.5", expected: false, name: "whitespace after equals" }, + { + accept: "text/event-stream;\u00a0q=0.5", + expected: false, + name: "non-HTTP parameter whitespace", + }, + { accept: "text/event-stream;q=0.1234", expected: false, name: "too many q digits" }, + { accept: "text/event-stream;q=1.001", expected: false, name: "qvalue above one" }, + { accept: "text/event-stream;q=1e0", expected: false, name: "exponent qvalue" }, + { accept: 'text/event-stream;q="0.5"', expected: false, name: "quoted qvalue" }, + { accept: "text/event-stream;q=0.5;q=1", expected: false, name: "duplicate q parameter" }, + { + accept: 'text/event-stream;q=0.5;legacy;note="quoted,comma;semicolon"', + expected: false, + name: "obsolete bare Accept extension after q", + }, + { + accept: 'text/event-stream; note="unterminated', + expected: false, + name: "unterminated quoted parameter", + }, + ])("returns $expected for $name", ({ accept, expected, representation }) => { + expect(hasExplicitAcceptableMediaRange(accept, representation ?? SSE_CONTENT_TYPE)).toBe( + expected, + ); + }); +}); + describe("session history HTTP endpoints", () => { + test("uses SSE only for an explicit acceptable event-stream media range", async () => { + const expectedText = "accept negotiation sentinel"; + await seedSession({ text: expectedText }); + await withGatewayHarness(async (harness) => { + const cases = [ + { accept: "text/event-stream", expected: "sse" }, + { accept: "TEXT/EVENT-STREAM", expected: "sse" }, + { accept: " text/event-stream ", expected: "sse" }, + { accept: "text/event-stream;", expected: "sse" }, + { accept: "text/event-stream; ; q=0.5;", expected: "sse" }, + { accept: "text/event-stream; charset=utf-8", expected: "sse" }, + { + accept: 'text/event-stream; note="quoted,comma;semicolon\\\"quote"; q=0.5', + expected: "json", + }, + { accept: "text/event-stream;q=0.001", expected: "sse" }, + { accept: "text/event-stream;Q=1.000", expected: "sse" }, + { accept: "text/event-stream;q=0, text/event-stream;q=0.5", expected: "sse" }, + { + accept: "text/event-stream;q=1, text/event-stream;charset=utf-8;q=0", + expected: "json", + }, + { + accept: "text/event-stream;q=0, text/event-stream;charset=utf-8;q=0.5", + expected: "sse", + }, + { accept: "text/event-stream;q=0.5;charset=utf-8", expected: "sse" }, + { accept: "text/event-stream;q=1;charset=utf-16", expected: "json" }, + { accept: "text/event-stream;charset=utf-16", expected: "json" }, + { accept: "text/event-streaming", expected: "json" }, + { accept: "text/event-streamx", expected: "json" }, + { accept: 'application/json; note="text/event-stream"', expected: "json" }, + { accept: "text/*", expected: "json" }, + { accept: "*/*", expected: "json" }, + { accept: "text/event-stream;q=0", expected: "json" }, + { accept: "text/event-stream;q=0, */*;q=1", expected: "json" }, + { accept: "text/event-stream;q=0.1234", expected: "json" }, + { accept: "text/event-stream;q =0.5", expected: "json" }, + { accept: "text/event-stream;q= 0.5", expected: "json" }, + { accept: "text/event-stream;\u00a0q=0.5", expected: "json" }, + { + accept: 'text/event-stream;q=0.5;legacy;note="quoted,comma;semicolon"', + expected: "json", + }, + ] as const; + + for (const testCase of cases) { + const response = await fetchSessionHistory(harness.port, "agent:main:main", { + headers: { Accept: testCase.accept }, + }); + expect(response.status, testCase.accept).toBe(200); + const contentType = response.headers.get("content-type") ?? ""; + if (testCase.expected === "sse") { + expect(contentType, testCase.accept).toContain("text/event-stream"); + const reader = response.body?.getReader(); + expect(reader, testCase.accept).toBeDefined(); + const event = await readSseEvent(reader!, { buffer: "" }); + expect(event.event, testCase.accept).toBe("history"); + expect( + (event.data as SessionHistoryBody).messages?.[0]?.content?.[0]?.text, + testCase.accept, + ).toBe(expectedText); + await reader!.cancel(); + continue; + } + expect(contentType, testCase.accept).toContain("application/json"); + const body = (await response.json()) as SessionHistoryBody; + expect(body.messages?.[0]?.content?.[0]?.text, testCase.accept).toBe(expectedText); + } + }); + }); + test("returns session history over direct REST", async () => { await seedSession({ text: "hello from history" }); await withGatewayHarness(async (harness) => { diff --git a/src/gateway/sessions-history-http.ts b/src/gateway/sessions-history-http.ts index 9980b18a8ad9..9b2296224fd5 100644 --- a/src/gateway/sessions-history-http.ts +++ b/src/gateway/sessions-history-http.ts @@ -3,10 +3,7 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import { parseStrictPositiveInteger } from "@openclaw/normalization-core/number-coercion"; import { err, ok, type Result } from "@openclaw/normalization-core/result"; -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "@openclaw/normalization-core/string-coerce"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { getRuntimeConfig } from "../config/io.js"; import { isSessionTranscriptProjectionUnavailableError } from "../config/sessions/session-accessor.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; @@ -20,7 +17,9 @@ import { sendJson, sendMethodNotAllowed, setSseHeaders, + SSE_CONTENT_TYPE, } from "./http-common.js"; +import { hasExplicitAcceptableMediaRange } from "./http-media-range.js"; import { authorizeScopedGatewayHttpRequestOrReply, checkGatewayHttpRequestAuth, @@ -62,8 +61,7 @@ function resolveSessionHistoryPath(req: IncomingMessage): string | null { } function shouldStreamSse(req: IncomingMessage): boolean { - const accept = normalizeLowercaseStringOrEmpty(getHeader(req, "accept")); - return accept.includes("text/event-stream"); + return hasExplicitAcceptableMediaRange(getHeader(req, "accept"), SSE_CONTENT_TYPE); } function getRequestUrl(req: IncomingMessage): URL {