fix: refresh Copilot token before expiry and retry on auth errors

GitHub Copilot API tokens expire after ~30 minutes. When OpenClaw spawns a long-running subagent using Copilot as the provider, the token would expire mid-session with no recovery mechanism, causing 401 auth errors. This commit adds: - Periodic token refresh scheduled 5 minutes before expiry - Auth error detection with automatic token refresh and single retry - Proper timer cleanup on session shutdown to prevent leaks The implementation uses a per-attempt retry flag to ensure each auth error can trigger one refresh+retry cycle without creating infinite retry loops. 🤖 AI-assisted: This fix was developed with GitHub Copilot CLI assistance. Testing: Fully tested with 3 new unit tests covering auth retry, retry reset, and timer cleanup scenarios. All 11 auth rotation tests pass.
2026-03-12 07:20:45 +00:00 · 2026-02-04 09:13:59 -03:00
parent e54ddf6161
commit 2dcd2f9094
2 changed files with 410 additions and 0 deletions
--- a/src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts
+++ b/src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts
@@ -8,11 +8,17 @@ import type { AuthProfileFailureReason } from "./auth-profiles.js";
 import type { EmbeddedRunAttemptResult } from "./pi-embedded-runner/run/types.js";

 const runEmbeddedAttemptMock = vi.fn<(params: unknown) => Promise<EmbeddedRunAttemptResult>>();
+const resolveCopilotApiTokenMock = vi.fn();

 vi.mock("./pi-embedded-runner/run/attempt.js", () => ({
  runEmbeddedAttempt: (params: unknown) => runEmbeddedAttemptMock(params),
 }));

+vi.mock("../providers/github-copilot-token.js", () => ({
+  DEFAULT_COPILOT_API_BASE_URL: "https://api.individual.githubcopilot.com",
+  resolveCopilotApiToken: (...args: unknown[]) => resolveCopilotApiTokenMock(...args),
+}));
+
 vi.mock("./pi-embedded-runner/compact.js", () => ({
  compactEmbeddedPiSessionDirect: vi.fn(async () => {
    throw new Error("compact should not run in auth profile rotation tests");
@@ -36,6 +42,7 @@ beforeAll(async () => {
 beforeEach(() => {
  vi.useRealTimers();
  runEmbeddedAttemptMock.mockClear();
+  resolveCopilotApiTokenMock.mockReset();
 });

 const baseUsage = {
@@ -148,6 +155,31 @@ const makeAgentOverrideOnlyFallbackConfig = (agentId: string): OpenClawConfig =>
    },
  }) satisfies OpenClawConfig;

+const copilotModelId = "gpt-4o";
+
+const makeCopilotConfig = (): OpenClawConfig =>
+  ({
+    models: {
+      providers: {
+        "github-copilot": {
+          api: "openai-responses",
+          baseUrl: "https://api.copilot.example",
+          models: [
+            {
+              id: copilotModelId,
+              name: "Copilot GPT-4o",
+              reasoning: false,
+              input: ["text"],
+              cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+              contextWindow: 16_000,
+              maxTokens: 2048,
+            },
+          ],
+        },
+      },
+    },
+  }) satisfies OpenClawConfig;
+
 const writeAuthStore = async (
  agentDir: string,
  opts?: {
@@ -184,6 +216,20 @@ const writeAuthStore = async (
  await fs.writeFile(authPath, JSON.stringify(payload));
 };

+const writeCopilotAuthStore = async (agentDir: string, token = "gh-token") => {
+  const authPath = path.join(agentDir, "auth-profiles.json");
+  const payload = {
+    version: 1,
+    profiles: {
+      "github-copilot:github": { type: "token", provider: "github-copilot", token },
+    },
+  };
+  await fs.writeFile(authPath, JSON.stringify(payload));
+};
+
+const buildCopilotAssistant = (overrides: Partial<AssistantMessage> = {}) =>
+  buildAssistant({ provider: "github-copilot", model: copilotModelId, ...overrides });
+
 const mockFailedThenSuccessfulAttempt = (errorMessage = "rate limit") => {
  runEmbeddedAttemptMock
    .mockResolvedValueOnce(
@@ -375,6 +421,215 @@ async function runTurnWithCooldownSeed(params: {
 }

 describe("runEmbeddedPiAgent auth profile rotation", () => {
+  it("refreshes copilot token after auth error and retries once", async () => {
+    const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-"));
+    const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
+    vi.useFakeTimers();
+    try {
+      await writeCopilotAuthStore(agentDir);
+      const now = Date.now();
+      vi.setSystemTime(now);
+
+      resolveCopilotApiTokenMock
+        .mockResolvedValueOnce({
+          token: "copilot-initial",
+          expiresAt: now + 2 * 60 * 1000,
+          source: "mock",
+          baseUrl: "https://api.copilot.example",
+        })
+        .mockResolvedValueOnce({
+          token: "copilot-refresh",
+          expiresAt: now + 60 * 60 * 1000,
+          source: "mock",
+          baseUrl: "https://api.copilot.example",
+        });
+
+      runEmbeddedAttemptMock
+        .mockResolvedValueOnce(
+          makeAttempt({
+            assistantTexts: [],
+            lastAssistant: buildCopilotAssistant({
+              stopReason: "error",
+              errorMessage: "unauthorized",
+            }),
+          }),
+        )
+        .mockResolvedValueOnce(
+          makeAttempt({
+            assistantTexts: ["ok"],
+            lastAssistant: buildCopilotAssistant({
+              stopReason: "stop",
+              content: [{ type: "text", text: "ok" }],
+            }),
+          }),
+        );
+
+      await runEmbeddedPiAgent({
+        sessionId: "session:test",
+        sessionKey: "agent:test:copilot-auth-error",
+        sessionFile: path.join(workspaceDir, "session.jsonl"),
+        workspaceDir,
+        agentDir,
+        config: makeCopilotConfig(),
+        prompt: "hello",
+        provider: "github-copilot",
+        model: copilotModelId,
+        authProfileIdSource: "auto",
+        timeoutMs: 5_000,
+        runId: "run:copilot-auth-error",
+      });
+
+      expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(2);
+      expect(resolveCopilotApiTokenMock).toHaveBeenCalledTimes(2);
+    } finally {
+      vi.useRealTimers();
+      await fs.rm(agentDir, { recursive: true, force: true });
+      await fs.rm(workspaceDir, { recursive: true, force: true });
+    }
+  });
+
+  it("allows another auth refresh after a successful retry", async () => {
+    const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-"));
+    const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
+    vi.useFakeTimers();
+    try {
+      await writeCopilotAuthStore(agentDir);
+      const now = Date.now();
+      vi.setSystemTime(now);
+
+      resolveCopilotApiTokenMock
+        .mockResolvedValueOnce({
+          token: "copilot-initial",
+          expiresAt: now + 2 * 60 * 1000,
+          source: "mock",
+          baseUrl: "https://api.copilot.example",
+        })
+        .mockResolvedValueOnce({
+          token: "copilot-refresh-1",
+          expiresAt: now + 4 * 60 * 1000,
+          source: "mock",
+          baseUrl: "https://api.copilot.example",
+        })
+        .mockResolvedValueOnce({
+          token: "copilot-refresh-2",
+          expiresAt: now + 40 * 60 * 1000,
+          source: "mock",
+          baseUrl: "https://api.copilot.example",
+        });
+
+      runEmbeddedAttemptMock
+        .mockResolvedValueOnce(
+          makeAttempt({
+            assistantTexts: [],
+            lastAssistant: buildCopilotAssistant({
+              stopReason: "error",
+              errorMessage: "401 unauthorized",
+            }),
+          }),
+        )
+        .mockResolvedValueOnce(
+          makeAttempt({
+            promptError: new Error("supported values are: low, medium"),
+          }),
+        )
+        .mockResolvedValueOnce(
+          makeAttempt({
+            assistantTexts: [],
+            lastAssistant: buildCopilotAssistant({
+              stopReason: "error",
+              errorMessage: "token has expired",
+            }),
+          }),
+        )
+        .mockResolvedValueOnce(
+          makeAttempt({
+            assistantTexts: ["ok"],
+            lastAssistant: buildCopilotAssistant({
+              stopReason: "stop",
+              content: [{ type: "text", text: "ok" }],
+            }),
+          }),
+        );
+
+      await runEmbeddedPiAgent({
+        sessionId: "session:test",
+        sessionKey: "agent:test:copilot-auth-repeat",
+        sessionFile: path.join(workspaceDir, "session.jsonl"),
+        workspaceDir,
+        agentDir,
+        config: makeCopilotConfig(),
+        prompt: "hello",
+        provider: "github-copilot",
+        model: copilotModelId,
+        authProfileIdSource: "auto",
+        timeoutMs: 5_000,
+        runId: "run:copilot-auth-repeat",
+      });
+
+      expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(4);
+      expect(resolveCopilotApiTokenMock).toHaveBeenCalledTimes(3);
+    } finally {
+      vi.useRealTimers();
+      await fs.rm(agentDir, { recursive: true, force: true });
+      await fs.rm(workspaceDir, { recursive: true, force: true });
+    }
+  });
+
+  it("does not reschedule copilot refresh after shutdown", async () => {
+    const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-"));
+    const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
+    vi.useFakeTimers();
+    try {
+      await writeCopilotAuthStore(agentDir);
+      const now = Date.now();
+      vi.setSystemTime(now);
+
+      resolveCopilotApiTokenMock.mockResolvedValue({
+        token: "copilot-initial",
+        expiresAt: now + 60 * 60 * 1000,
+        source: "mock",
+        baseUrl: "https://api.copilot.example",
+      });
+
+      runEmbeddedAttemptMock.mockResolvedValueOnce(
+        makeAttempt({
+          assistantTexts: ["ok"],
+          lastAssistant: buildCopilotAssistant({
+            stopReason: "stop",
+            content: [{ type: "text", text: "ok" }],
+          }),
+        }),
+      );
+
+      const runPromise = runEmbeddedPiAgent({
+        sessionId: "session:test",
+        sessionKey: "agent:test:copilot-shutdown",
+        sessionFile: path.join(workspaceDir, "session.jsonl"),
+        workspaceDir,
+        agentDir,
+        config: makeCopilotConfig(),
+        prompt: "hello",
+        provider: "github-copilot",
+        model: copilotModelId,
+        authProfileIdSource: "auto",
+        timeoutMs: 5_000,
+        runId: "run:copilot-shutdown",
+      });
+
+      await vi.advanceTimersByTimeAsync(1);
+      await runPromise;
+      const refreshCalls = resolveCopilotApiTokenMock.mock.calls.length;
+
+      await vi.advanceTimersByTimeAsync(2 * 60 * 1000);
+
+      expect(resolveCopilotApiTokenMock.mock.calls.length).toBe(refreshCalls);
+    } finally {
+      vi.useRealTimers();
+      await fs.rm(agentDir, { recursive: true, force: true });
+      await fs.rm(workspaceDir, { recursive: true, force: true });
+    }
+  });
+
  it("rotates for auto-pinned profiles across retryable stream failures", async () => {
    const { usageStats } = await runAutoPinnedRotationCase({
      errorMessage: "rate limit",
--- a/src/agents/pi-embedded-runner/run.ts
+++ b/src/agents/pi-embedded-runner/run.ts
@@ -66,6 +66,17 @@ import { describeUnknownError } from "./utils.js";

 type ApiKeyInfo = ResolvedProviderAuth;

+type CopilotTokenState = {
+  githubToken: string;
+  expiresAt: number;
+  refreshTimer?: ReturnType<typeof setTimeout>;
+  refreshInFlight?: Promise<void>;
+};
+
+const COPILOT_REFRESH_MARGIN_MS = 5 * 60 * 1000;
+const COPILOT_REFRESH_RETRY_MS = 60 * 1000;
+const COPILOT_REFRESH_MIN_DELAY_MS = 5 * 1000;
+
 // Avoid Anthropic's refusal test token poisoning session transcripts.
 const ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL = "ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL";
 const ANTHROPIC_MAGIC_STRING_REPLACEMENT = "ANTHROPIC MAGIC STRING TRIGGER REFUSAL (redacted)";
@@ -365,6 +376,105 @@ export async function runEmbeddedPiAgent(
      const attemptedThinking = new Set<ThinkLevel>();
      let apiKeyInfo: ApiKeyInfo | null = null;
      let lastProfileId: string | undefined;
+      const copilotTokenState: CopilotTokenState | null =
+        model.provider === "github-copilot" ? { githubToken: "", expiresAt: 0 } : null;
+      let copilotRefreshCancelled = false;
+      const hasCopilotGithubToken = () => Boolean(copilotTokenState?.githubToken.trim());
+
+      const clearCopilotRefreshTimer = () => {
+        if (!copilotTokenState?.refreshTimer) {
+          return;
+        }
+        clearTimeout(copilotTokenState.refreshTimer);
+        copilotTokenState.refreshTimer = undefined;
+      };
+
+      const stopCopilotRefreshTimer = () => {
+        if (!copilotTokenState) {
+          return;
+        }
+        copilotRefreshCancelled = true;
+        clearCopilotRefreshTimer();
+      };
+
+      const refreshCopilotToken = async (reason: string): Promise<void> => {
+        if (!copilotTokenState) {
+          return;
+        }
+        if (copilotTokenState.refreshInFlight) {
+          await copilotTokenState.refreshInFlight;
+          return;
+        }
+        const { resolveCopilotApiToken } = await import("../../providers/github-copilot-token.js");
+        copilotTokenState.refreshInFlight = (async () => {
+          const githubToken = copilotTokenState.githubToken.trim();
+          if (!githubToken) {
+            throw new Error("Copilot refresh requires a GitHub token.");
+          }
+          log.debug(`Refreshing GitHub Copilot token (${reason})...`);
+          const copilotToken = await resolveCopilotApiToken({
+            githubToken,
+          });
+          authStorage.setRuntimeApiKey(model.provider, copilotToken.token);
+          copilotTokenState.expiresAt = copilotToken.expiresAt;
+          const remaining = copilotToken.expiresAt - Date.now();
+          log.debug(
+            `Copilot token refreshed; expires in ${Math.max(0, Math.floor(remaining / 1000))}s.`,
+          );
+        })()
+          .catch((err) => {
+            log.warn(`Copilot token refresh failed: ${describeUnknownError(err)}`);
+            throw err;
+          })
+          .finally(() => {
+            copilotTokenState.refreshInFlight = undefined;
+          });
+        await copilotTokenState.refreshInFlight;
+      };
+
+      const scheduleCopilotRefresh = (): void => {
+        if (!copilotTokenState || copilotRefreshCancelled) {
+          return;
+        }
+        if (!hasCopilotGithubToken()) {
+          log.warn("Skipping Copilot refresh scheduling; GitHub token missing.");
+          return;
+        }
+        clearCopilotRefreshTimer();
+        const now = Date.now();
+        const refreshAt = copilotTokenState.expiresAt - COPILOT_REFRESH_MARGIN_MS;
+        const delayMs = Math.max(COPILOT_REFRESH_MIN_DELAY_MS, refreshAt - now);
+        const timer = setTimeout(() => {
+          if (copilotRefreshCancelled) {
+            return;
+          }
+          refreshCopilotToken("scheduled")
+            .then(() => scheduleCopilotRefresh())
+            .catch(() => {
+              if (copilotRefreshCancelled) {
+                return;
+              }
+              const retryTimer = setTimeout(() => {
+                if (copilotRefreshCancelled) {
+                  return;
+                }
+                refreshCopilotToken("scheduled-retry")
+                  .then(() => scheduleCopilotRefresh())
+                  .catch(() => undefined);
+              }, COPILOT_REFRESH_RETRY_MS);
+              copilotTokenState.refreshTimer = retryTimer;
+              if (copilotRefreshCancelled) {
+                clearTimeout(retryTimer);
+                copilotTokenState.refreshTimer = undefined;
+              }
+            });
+        }, delayMs);
+        copilotTokenState.refreshTimer = timer;
+        if (copilotRefreshCancelled) {
+          clearTimeout(timer);
+          copilotTokenState.refreshTimer = undefined;
+        }
+      };

      const resolveAuthProfileFailoverReason = (params: {
        allInCooldown: boolean;
@@ -445,6 +555,11 @@ export async function runEmbeddedPiAgent(
            githubToken: apiKeyInfo.apiKey,
          });
          authStorage.setRuntimeApiKey(model.provider, copilotToken.token);
+          if (copilotTokenState) {
+            copilotTokenState.githubToken = apiKeyInfo.apiKey;
+            copilotTokenState.expiresAt = copilotToken.expiresAt;
+            scheduleCopilotRefresh();
+          }
        } else {
          authStorage.setRuntimeApiKey(model.provider, apiKeyInfo.apiKey);
        }
@@ -508,6 +623,28 @@ export async function runEmbeddedPiAgent(
        }
      }

+      const maybeRefreshCopilotForAuthError = async (
+        errorText: string,
+        retried: boolean,
+      ): Promise<boolean> => {
+        if (!copilotTokenState || retried) {
+          return false;
+        }
+        if (!isFailoverErrorMessage(errorText)) {
+          return false;
+        }
+        if (classifyFailoverReason(errorText) !== "auth") {
+          return false;
+        }
+        try {
+          await refreshCopilotToken("auth-error");
+          scheduleCopilotRefresh();
+          return true;
+        } catch {
+          return false;
+        }
+      };
+
      const MAX_OVERFLOW_COMPACTION_ATTEMPTS = 3;
      const MAX_RUN_LOOP_ITERATIONS = resolveMaxRunRetryIterations(profileCandidates.length);
      let overflowCompactionAttempts = 0;
@@ -535,6 +672,7 @@ export async function runEmbeddedPiAgent(
        });
      };
      try {
+        let authRetryPending = false;
        while (true) {
          if (runLoopIterations >= MAX_RUN_LOOP_ITERATIONS) {
            const message =
@@ -566,6 +704,8 @@ export async function runEmbeddedPiAgent(
            };
          }
          runLoopIterations += 1;
+          const copilotAuthRetry = authRetryPending;
+          authRetryPending = false;
          attemptedThinking.add(thinkLevel);
          await fs.mkdir(resolvedWorkspace, { recursive: true });

@@ -852,6 +992,10 @@ export async function runEmbeddedPiAgent(

          if (promptError && !aborted) {
            const errorText = describeUnknownError(promptError);
+            if (await maybeRefreshCopilotForAuthError(errorText, copilotAuthRetry)) {
+              authRetryPending = true;
+              continue;
+            }
            // Handle role ordering errors with a user-friendly message
            if (/incorrect role information|roles must alternate/i.test(errorText)) {
              return {
@@ -960,6 +1104,16 @@ export async function runEmbeddedPiAgent(
          const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError;
          const imageDimensionError = parseImageDimensionError(lastAssistant?.errorMessage ?? "");

+          if (
+            authFailure &&
+            (await maybeRefreshCopilotForAuthError(
+              lastAssistant?.errorMessage ?? "",
+              copilotAuthRetry,
+            ))
+          ) {
+            authRetryPending = true;
+            continue;
+          }
          if (imageDimensionError && lastProfileId) {
            const details = [
              imageDimensionError.messageIndex !== undefined
@@ -1157,6 +1311,7 @@ export async function runEmbeddedPiAgent(
          };
        }
      } finally {
+        stopCopilotRefreshTimer();
        process.chdir(prevCwd);
      }
    }),