diff --git a/src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts b/src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts index ca66ad4c7f7..cf56036c3ea 100644 --- a/src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts +++ b/src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts @@ -8,11 +8,17 @@ import type { AuthProfileFailureReason } from "./auth-profiles.js"; import type { EmbeddedRunAttemptResult } from "./pi-embedded-runner/run/types.js"; const runEmbeddedAttemptMock = vi.fn<(params: unknown) => Promise>(); +const resolveCopilotApiTokenMock = vi.fn(); vi.mock("./pi-embedded-runner/run/attempt.js", () => ({ runEmbeddedAttempt: (params: unknown) => runEmbeddedAttemptMock(params), })); +vi.mock("../providers/github-copilot-token.js", () => ({ + DEFAULT_COPILOT_API_BASE_URL: "https://api.individual.githubcopilot.com", + resolveCopilotApiToken: (...args: unknown[]) => resolveCopilotApiTokenMock(...args), +})); + vi.mock("./pi-embedded-runner/compact.js", () => ({ compactEmbeddedPiSessionDirect: vi.fn(async () => { throw new Error("compact should not run in auth profile rotation tests"); @@ -36,6 +42,7 @@ beforeAll(async () => { beforeEach(() => { vi.useRealTimers(); runEmbeddedAttemptMock.mockClear(); + resolveCopilotApiTokenMock.mockReset(); }); const baseUsage = { @@ -148,6 +155,31 @@ const makeAgentOverrideOnlyFallbackConfig = (agentId: string): OpenClawConfig => }, }) satisfies OpenClawConfig; +const copilotModelId = "gpt-4o"; + +const makeCopilotConfig = (): OpenClawConfig => + ({ + models: { + providers: { + "github-copilot": { + api: "openai-responses", + baseUrl: "https://api.copilot.example", + models: [ + { + id: copilotModelId, + name: "Copilot GPT-4o", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 16_000, + maxTokens: 2048, + }, + ], + }, + }, + }, + }) satisfies OpenClawConfig; + const writeAuthStore = async ( agentDir: string, opts?: { @@ -184,6 +216,20 @@ const writeAuthStore = async ( await fs.writeFile(authPath, JSON.stringify(payload)); }; +const writeCopilotAuthStore = async (agentDir: string, token = "gh-token") => { + const authPath = path.join(agentDir, "auth-profiles.json"); + const payload = { + version: 1, + profiles: { + "github-copilot:github": { type: "token", provider: "github-copilot", token }, + }, + }; + await fs.writeFile(authPath, JSON.stringify(payload)); +}; + +const buildCopilotAssistant = (overrides: Partial = {}) => + buildAssistant({ provider: "github-copilot", model: copilotModelId, ...overrides }); + const mockFailedThenSuccessfulAttempt = (errorMessage = "rate limit") => { runEmbeddedAttemptMock .mockResolvedValueOnce( @@ -375,6 +421,215 @@ async function runTurnWithCooldownSeed(params: { } describe("runEmbeddedPiAgent auth profile rotation", () => { + it("refreshes copilot token after auth error and retries once", async () => { + const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-")); + vi.useFakeTimers(); + try { + await writeCopilotAuthStore(agentDir); + const now = Date.now(); + vi.setSystemTime(now); + + resolveCopilotApiTokenMock + .mockResolvedValueOnce({ + token: "copilot-initial", + expiresAt: now + 2 * 60 * 1000, + source: "mock", + baseUrl: "https://api.copilot.example", + }) + .mockResolvedValueOnce({ + token: "copilot-refresh", + expiresAt: now + 60 * 60 * 1000, + source: "mock", + baseUrl: "https://api.copilot.example", + }); + + runEmbeddedAttemptMock + .mockResolvedValueOnce( + makeAttempt({ + assistantTexts: [], + lastAssistant: buildCopilotAssistant({ + stopReason: "error", + errorMessage: "unauthorized", + }), + }), + ) + .mockResolvedValueOnce( + makeAttempt({ + assistantTexts: ["ok"], + lastAssistant: buildCopilotAssistant({ + stopReason: "stop", + content: [{ type: "text", text: "ok" }], + }), + }), + ); + + await runEmbeddedPiAgent({ + sessionId: "session:test", + sessionKey: "agent:test:copilot-auth-error", + sessionFile: path.join(workspaceDir, "session.jsonl"), + workspaceDir, + agentDir, + config: makeCopilotConfig(), + prompt: "hello", + provider: "github-copilot", + model: copilotModelId, + authProfileIdSource: "auto", + timeoutMs: 5_000, + runId: "run:copilot-auth-error", + }); + + expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(2); + expect(resolveCopilotApiTokenMock).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + await fs.rm(agentDir, { recursive: true, force: true }); + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("allows another auth refresh after a successful retry", async () => { + const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-")); + vi.useFakeTimers(); + try { + await writeCopilotAuthStore(agentDir); + const now = Date.now(); + vi.setSystemTime(now); + + resolveCopilotApiTokenMock + .mockResolvedValueOnce({ + token: "copilot-initial", + expiresAt: now + 2 * 60 * 1000, + source: "mock", + baseUrl: "https://api.copilot.example", + }) + .mockResolvedValueOnce({ + token: "copilot-refresh-1", + expiresAt: now + 4 * 60 * 1000, + source: "mock", + baseUrl: "https://api.copilot.example", + }) + .mockResolvedValueOnce({ + token: "copilot-refresh-2", + expiresAt: now + 40 * 60 * 1000, + source: "mock", + baseUrl: "https://api.copilot.example", + }); + + runEmbeddedAttemptMock + .mockResolvedValueOnce( + makeAttempt({ + assistantTexts: [], + lastAssistant: buildCopilotAssistant({ + stopReason: "error", + errorMessage: "401 unauthorized", + }), + }), + ) + .mockResolvedValueOnce( + makeAttempt({ + promptError: new Error("supported values are: low, medium"), + }), + ) + .mockResolvedValueOnce( + makeAttempt({ + assistantTexts: [], + lastAssistant: buildCopilotAssistant({ + stopReason: "error", + errorMessage: "token has expired", + }), + }), + ) + .mockResolvedValueOnce( + makeAttempt({ + assistantTexts: ["ok"], + lastAssistant: buildCopilotAssistant({ + stopReason: "stop", + content: [{ type: "text", text: "ok" }], + }), + }), + ); + + await runEmbeddedPiAgent({ + sessionId: "session:test", + sessionKey: "agent:test:copilot-auth-repeat", + sessionFile: path.join(workspaceDir, "session.jsonl"), + workspaceDir, + agentDir, + config: makeCopilotConfig(), + prompt: "hello", + provider: "github-copilot", + model: copilotModelId, + authProfileIdSource: "auto", + timeoutMs: 5_000, + runId: "run:copilot-auth-repeat", + }); + + expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(4); + expect(resolveCopilotApiTokenMock).toHaveBeenCalledTimes(3); + } finally { + vi.useRealTimers(); + await fs.rm(agentDir, { recursive: true, force: true }); + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("does not reschedule copilot refresh after shutdown", async () => { + const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-")); + vi.useFakeTimers(); + try { + await writeCopilotAuthStore(agentDir); + const now = Date.now(); + vi.setSystemTime(now); + + resolveCopilotApiTokenMock.mockResolvedValue({ + token: "copilot-initial", + expiresAt: now + 60 * 60 * 1000, + source: "mock", + baseUrl: "https://api.copilot.example", + }); + + runEmbeddedAttemptMock.mockResolvedValueOnce( + makeAttempt({ + assistantTexts: ["ok"], + lastAssistant: buildCopilotAssistant({ + stopReason: "stop", + content: [{ type: "text", text: "ok" }], + }), + }), + ); + + const runPromise = runEmbeddedPiAgent({ + sessionId: "session:test", + sessionKey: "agent:test:copilot-shutdown", + sessionFile: path.join(workspaceDir, "session.jsonl"), + workspaceDir, + agentDir, + config: makeCopilotConfig(), + prompt: "hello", + provider: "github-copilot", + model: copilotModelId, + authProfileIdSource: "auto", + timeoutMs: 5_000, + runId: "run:copilot-shutdown", + }); + + await vi.advanceTimersByTimeAsync(1); + await runPromise; + const refreshCalls = resolveCopilotApiTokenMock.mock.calls.length; + + await vi.advanceTimersByTimeAsync(2 * 60 * 1000); + + expect(resolveCopilotApiTokenMock.mock.calls.length).toBe(refreshCalls); + } finally { + vi.useRealTimers(); + await fs.rm(agentDir, { recursive: true, force: true }); + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + it("rotates for auto-pinned profiles across retryable stream failures", async () => { const { usageStats } = await runAutoPinnedRotationCase({ errorMessage: "rate limit", diff --git a/src/agents/pi-embedded-runner/run.ts b/src/agents/pi-embedded-runner/run.ts index 06df4cb4351..9d440bda6eb 100644 --- a/src/agents/pi-embedded-runner/run.ts +++ b/src/agents/pi-embedded-runner/run.ts @@ -66,6 +66,17 @@ import { describeUnknownError } from "./utils.js"; type ApiKeyInfo = ResolvedProviderAuth; +type CopilotTokenState = { + githubToken: string; + expiresAt: number; + refreshTimer?: ReturnType; + refreshInFlight?: Promise; +}; + +const COPILOT_REFRESH_MARGIN_MS = 5 * 60 * 1000; +const COPILOT_REFRESH_RETRY_MS = 60 * 1000; +const COPILOT_REFRESH_MIN_DELAY_MS = 5 * 1000; + // Avoid Anthropic's refusal test token poisoning session transcripts. const ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL = "ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL"; const ANTHROPIC_MAGIC_STRING_REPLACEMENT = "ANTHROPIC MAGIC STRING TRIGGER REFUSAL (redacted)"; @@ -365,6 +376,105 @@ export async function runEmbeddedPiAgent( const attemptedThinking = new Set(); let apiKeyInfo: ApiKeyInfo | null = null; let lastProfileId: string | undefined; + const copilotTokenState: CopilotTokenState | null = + model.provider === "github-copilot" ? { githubToken: "", expiresAt: 0 } : null; + let copilotRefreshCancelled = false; + const hasCopilotGithubToken = () => Boolean(copilotTokenState?.githubToken.trim()); + + const clearCopilotRefreshTimer = () => { + if (!copilotTokenState?.refreshTimer) { + return; + } + clearTimeout(copilotTokenState.refreshTimer); + copilotTokenState.refreshTimer = undefined; + }; + + const stopCopilotRefreshTimer = () => { + if (!copilotTokenState) { + return; + } + copilotRefreshCancelled = true; + clearCopilotRefreshTimer(); + }; + + const refreshCopilotToken = async (reason: string): Promise => { + if (!copilotTokenState) { + return; + } + if (copilotTokenState.refreshInFlight) { + await copilotTokenState.refreshInFlight; + return; + } + const { resolveCopilotApiToken } = await import("../../providers/github-copilot-token.js"); + copilotTokenState.refreshInFlight = (async () => { + const githubToken = copilotTokenState.githubToken.trim(); + if (!githubToken) { + throw new Error("Copilot refresh requires a GitHub token."); + } + log.debug(`Refreshing GitHub Copilot token (${reason})...`); + const copilotToken = await resolveCopilotApiToken({ + githubToken, + }); + authStorage.setRuntimeApiKey(model.provider, copilotToken.token); + copilotTokenState.expiresAt = copilotToken.expiresAt; + const remaining = copilotToken.expiresAt - Date.now(); + log.debug( + `Copilot token refreshed; expires in ${Math.max(0, Math.floor(remaining / 1000))}s.`, + ); + })() + .catch((err) => { + log.warn(`Copilot token refresh failed: ${describeUnknownError(err)}`); + throw err; + }) + .finally(() => { + copilotTokenState.refreshInFlight = undefined; + }); + await copilotTokenState.refreshInFlight; + }; + + const scheduleCopilotRefresh = (): void => { + if (!copilotTokenState || copilotRefreshCancelled) { + return; + } + if (!hasCopilotGithubToken()) { + log.warn("Skipping Copilot refresh scheduling; GitHub token missing."); + return; + } + clearCopilotRefreshTimer(); + const now = Date.now(); + const refreshAt = copilotTokenState.expiresAt - COPILOT_REFRESH_MARGIN_MS; + const delayMs = Math.max(COPILOT_REFRESH_MIN_DELAY_MS, refreshAt - now); + const timer = setTimeout(() => { + if (copilotRefreshCancelled) { + return; + } + refreshCopilotToken("scheduled") + .then(() => scheduleCopilotRefresh()) + .catch(() => { + if (copilotRefreshCancelled) { + return; + } + const retryTimer = setTimeout(() => { + if (copilotRefreshCancelled) { + return; + } + refreshCopilotToken("scheduled-retry") + .then(() => scheduleCopilotRefresh()) + .catch(() => undefined); + }, COPILOT_REFRESH_RETRY_MS); + copilotTokenState.refreshTimer = retryTimer; + if (copilotRefreshCancelled) { + clearTimeout(retryTimer); + copilotTokenState.refreshTimer = undefined; + } + }); + }, delayMs); + copilotTokenState.refreshTimer = timer; + if (copilotRefreshCancelled) { + clearTimeout(timer); + copilotTokenState.refreshTimer = undefined; + } + }; const resolveAuthProfileFailoverReason = (params: { allInCooldown: boolean; @@ -445,6 +555,11 @@ export async function runEmbeddedPiAgent( githubToken: apiKeyInfo.apiKey, }); authStorage.setRuntimeApiKey(model.provider, copilotToken.token); + if (copilotTokenState) { + copilotTokenState.githubToken = apiKeyInfo.apiKey; + copilotTokenState.expiresAt = copilotToken.expiresAt; + scheduleCopilotRefresh(); + } } else { authStorage.setRuntimeApiKey(model.provider, apiKeyInfo.apiKey); } @@ -508,6 +623,28 @@ export async function runEmbeddedPiAgent( } } + const maybeRefreshCopilotForAuthError = async ( + errorText: string, + retried: boolean, + ): Promise => { + if (!copilotTokenState || retried) { + return false; + } + if (!isFailoverErrorMessage(errorText)) { + return false; + } + if (classifyFailoverReason(errorText) !== "auth") { + return false; + } + try { + await refreshCopilotToken("auth-error"); + scheduleCopilotRefresh(); + return true; + } catch { + return false; + } + }; + const MAX_OVERFLOW_COMPACTION_ATTEMPTS = 3; const MAX_RUN_LOOP_ITERATIONS = resolveMaxRunRetryIterations(profileCandidates.length); let overflowCompactionAttempts = 0; @@ -535,6 +672,7 @@ export async function runEmbeddedPiAgent( }); }; try { + let authRetryPending = false; while (true) { if (runLoopIterations >= MAX_RUN_LOOP_ITERATIONS) { const message = @@ -566,6 +704,8 @@ export async function runEmbeddedPiAgent( }; } runLoopIterations += 1; + const copilotAuthRetry = authRetryPending; + authRetryPending = false; attemptedThinking.add(thinkLevel); await fs.mkdir(resolvedWorkspace, { recursive: true }); @@ -852,6 +992,10 @@ export async function runEmbeddedPiAgent( if (promptError && !aborted) { const errorText = describeUnknownError(promptError); + if (await maybeRefreshCopilotForAuthError(errorText, copilotAuthRetry)) { + authRetryPending = true; + continue; + } // Handle role ordering errors with a user-friendly message if (/incorrect role information|roles must alternate/i.test(errorText)) { return { @@ -960,6 +1104,16 @@ export async function runEmbeddedPiAgent( const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError; const imageDimensionError = parseImageDimensionError(lastAssistant?.errorMessage ?? ""); + if ( + authFailure && + (await maybeRefreshCopilotForAuthError( + lastAssistant?.errorMessage ?? "", + copilotAuthRetry, + )) + ) { + authRetryPending = true; + continue; + } if (imageDimensionError && lastProfileId) { const details = [ imageDimensionError.messageIndex !== undefined @@ -1157,6 +1311,7 @@ export async function runEmbeddedPiAgent( }; } } finally { + stopCopilotRefreshTimer(); process.chdir(prevCwd); } }),