From 3ecb713b00702c9e1ec0618f612f4bae94511e65 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 20 Apr 2026 15:07:47 +0100 Subject: [PATCH] perf: speed local checks and warm builds --- .github/instructions/copilot.instructions.md | 6 +- .github/workflows/ci.yml | 5 + .github/workflows/openclaw-npm-release.yml | 5 + AGENTS.md | 19 +- docs/ci.md | 41 ++-- docs/help/testing.md | 2 +- docs/reference/RELEASING.md | 2 + docs/reference/test.md | 1 + package.json | 2 + scripts/build-all.mjs | 221 ++++++++++++++++++ scripts/check.mjs | 10 +- .../reply/directive-handling.model.test.ts | 36 ++- test/helpers/channels/imessage-test-plugin.ts | 2 +- test/helpers/plugins/plugin-runtime-mock.ts | 4 +- test/scripts/build-all.test.ts | 103 ++++++++ 15 files changed, 411 insertions(+), 48 deletions(-) diff --git a/.github/instructions/copilot.instructions.md b/.github/instructions/copilot.instructions.md index 623ebcd8ccb..62bf9f8343b 100644 --- a/.github/instructions/copilot.instructions.md +++ b/.github/instructions/copilot.instructions.md @@ -49,14 +49,14 @@ - TypeScript (ESM), strict typing, avoid `any` - Keep files under ~700 LOC - extract helpers when larger - Colocated tests: `*.test.ts` next to source files -- Run `pnpm check` before commits (lint + format) -- Run `pnpm tsgo` for production type checking, or `pnpm tsgo:all` for production plus test types +- Run `pnpm check` before commits (production type check + lint + format) +- Run `pnpm check:test-types` when you need test type coverage, or `pnpm tsgo:all` for a full production plus test type sweep ## Stack & Commands - **Package manager**: pnpm (`pnpm install`) - **Dev**: `pnpm openclaw ...` or `pnpm dev` -- **Type-check**: `pnpm tsgo` (production), `pnpm tsgo:all` (production plus tests) +- **Type-check**: `pnpm tsgo` (core production), `pnpm tsgo:prod` (core + extension production), `pnpm check:test-types` (tests) - **Lint/format**: `pnpm check` - **Tests**: `pnpm test` - **Build**: `pnpm build` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e0fb79bb12..c2bdd6c965a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1158,6 +1158,11 @@ jobs: OPENCLAW_LOCAL_CHECK: "0" run: pnpm check + - name: Check test types + env: + OPENCLAW_LOCAL_CHECK: "0" + run: pnpm check:test-types + - name: Strict TS build smoke run: pnpm build:strict-smoke diff --git a/.github/workflows/openclaw-npm-release.yml b/.github/workflows/openclaw-npm-release.yml index 0ee021f9c94..0bff8fbafad 100644 --- a/.github/workflows/openclaw-npm-release.yml +++ b/.github/workflows/openclaw-npm-release.yml @@ -110,6 +110,11 @@ jobs: OPENCLAW_LOCAL_CHECK: "0" run: pnpm check + - name: Check test types + env: + OPENCLAW_LOCAL_CHECK: "0" + run: pnpm check:test-types + - name: Check architecture env: OPENCLAW_LOCAL_CHECK: "0" diff --git a/AGENTS.md b/AGENTS.md index eb188be05b7..7716c571362 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -127,16 +127,13 @@ - Node remains supported for running built output (`dist/*`) and production installs. - Mac packaging (dev): `scripts/package-mac-app.sh` defaults to current arch. - Type-check/build: `pnpm build` -- TypeScript checks are split by architecture boundary, with four normal lanes: - - `pnpm tsgo` / `pnpm tsgo:core`: core production roots (`src/`, `ui/`, `packages/`; no `extensions/` include roots). - - `pnpm tsgo:core:test`: core colocated tests. - - `pnpm tsgo:extensions`: bundled extension production graph. - - `pnpm tsgo:extensions:test`: bundled extension colocated tests. - - `pnpm tsgo:core:all`: core production + core test project references (`tsconfig.core.projects.json`). - - `pnpm tsgo:extensions:all`: extension production + extension test project references (`tsconfig.extensions.projects.json`). - - `pnpm tsgo:all`: every TypeScript graph above through the project-reference root (`tsconfig.projects.json`); this is what `pnpm check` runs. +- TypeScript checks are split by architecture boundary. Normal entrypoints: + - `pnpm tsgo`: fastest core production check (`src/`, `ui/`, `packages/`; no `extensions/` include roots). + - `pnpm tsgo:prod`: core + bundled extension production graphs; this is what `pnpm check` runs. + - `pnpm check:test-types` / `pnpm tsgo:test`: core + bundled extension test graphs. + - `pnpm tsgo:all`: every production + test graph through project references; use for full typecheck sweeps. + - Boundary/debug slices exist (`tsgo:core:test`, `tsgo:extensions:test`, `tsgo:core:all`, `tsgo:extensions:all`, `tsgo:test:src`, `tsgo:test:ui`, `tsgo:test:packages`), but do not present them as the normal flow. - `pnpm tsgo:profile [core-test|extensions-test|--all]`: profile fresh graph cost into `.artifacts/tsgo-profile/`. Diagnostic-only profile slices (`core-test-agents`, `core-test-non-agents`) exist for investigating agent graph cost; do not treat them as normal user-facing checks. - - Narrow aliases remain for local loops: `pnpm tsgo:test:src`, `pnpm tsgo:test:ui`, `pnpm tsgo:test:packages`. - Do not add `tsc --noEmit`, `typecheck`, or `check:types` lanes for repo type checking. Use `tsgo` graphs. `tsc` is allowed only when emitting declaration/package-boundary compatibility artifacts that `tsgo` does not replace. - Boundary rule: core must not know extension implementation details. Extensions hook into core through manifests, registries, capabilities, and public `openclaw/plugin-sdk/*` contracts. If you find core production code naming a specific extension, or a core test that is really testing extension-owned behavior, call it out and prefer moving coverage/logic to the owning extension or a generic contract test. - Lint/format: `pnpm check` @@ -152,8 +149,8 @@ - A local dev gate is the fast default loop, usually `pnpm check` plus any scoped test you actually need. - A landing gate is the broader bar before pushing `main`, usually `pnpm check`, `pnpm test`, and `pnpm build` when the touched surface can affect build output, packaging, lazy-loading/module boundaries, or published surfaces. - A CI gate is whatever the relevant workflow enforces for that lane (for example `check`, `check-additional`, `build-smoke`, or release validation). -- Local dev gate: prefer `pnpm check` for the normal edit loop. It runs typecheck and lint first, then parallelizes independent policy guards. It keeps the repo-architecture policy guards out of the default local loop. -- Timed local gate: use `pnpm check:timed` to see per-stage cost. Add `:architecture` only when investigating the CI architecture gate locally. +- Local dev gate: prefer `pnpm check` for the normal edit loop. It runs production typecheck, lint, and independent policy guards. It keeps test typecheck and repo-architecture policy guards out of the default local loop. +- Timed local gate: use `pnpm check:timed` to see per-stage cost. Use `pnpm check:timed:all-types` when investigating test typecheck cost. Add `:architecture` only when investigating the CI architecture gate locally. - CI architecture gate: `check-additional` enforces architecture and boundary policy guards that are intentionally kept out of the default local loop. - Formatting gate: the pre-commit hook runs targeted formatting on staged source files before `pnpm check`. If you want a repo-wide formatting-only preflight locally, run `pnpm format:check` explicitly. - If you need a fast commit loop, `FAST_COMMIT=1 git commit ...` skips the hook’s repo-wide `pnpm check`; targeted formatting/linting still runs, so use that only when you are deliberately covering the touched surface some other way. diff --git a/docs/ci.md b/docs/ci.md index 37a12acb98f..df00c6504d6 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -12,25 +12,25 @@ The CI runs on every push to `main` and every pull request. It uses smart scopin ## Job Overview -| Job | Purpose | When it runs | -| ------------------------ | --------------------------------------------------------------------------------------- | ----------------------------------- | -| `preflight` | Detect docs-only changes, changed scopes, changed extensions, and build the CI manifest | Always on non-draft pushes and PRs | -| `security-fast` | Private key detection, workflow audit via `zizmor`, production dependency audit | Always on non-draft pushes and PRs | -| `build-artifacts` | Build `dist/` and the Control UI once, upload reusable artifacts for downstream jobs | Node-relevant changes | -| `checks-fast-core` | Fast Linux correctness lanes such as bundled/plugin-contract/protocol checks | Node-relevant changes | -| `checks-node-extensions` | Full bundled-plugin test shards across the extension suite | Node-relevant changes | -| `checks-node-core-test` | Core Node test shards, excluding channel, bundled, contract, and extension lanes | Node-relevant changes | -| `extension-fast` | Focused tests for only the changed bundled plugins | When extension changes are detected | -| `check` | Main local gate in CI: `pnpm check` plus `pnpm build:strict-smoke` | Node-relevant changes | -| `check-additional` | Architecture, boundary, import-cycle guards plus the gateway watch regression harness | Node-relevant changes | -| `build-smoke` | Built-CLI smoke tests and startup-memory smoke | Node-relevant changes | -| `checks` | Remaining Linux Node lanes: channel tests and push-only Node 22 compatibility | Node-relevant changes | -| `check-docs` | Docs formatting, lint, and broken-link checks | Docs changed | -| `skills-python` | Ruff + pytest for Python-backed skills | Python-skill-relevant changes | -| `checks-windows` | Windows-specific test lanes | Windows-relevant changes | -| `macos-node` | macOS TypeScript test lane using the shared built artifacts | macOS-relevant changes | -| `macos-swift` | Swift lint, build, and tests for the macOS app | macOS-relevant changes | -| `android` | Android build and test matrix | Android-relevant changes | +| Job | Purpose | When it runs | +| ------------------------ | ------------------------------------------------------------------------------------------- | ----------------------------------- | +| `preflight` | Detect docs-only changes, changed scopes, changed extensions, and build the CI manifest | Always on non-draft pushes and PRs | +| `security-fast` | Private key detection, workflow audit via `zizmor`, production dependency audit | Always on non-draft pushes and PRs | +| `build-artifacts` | Build `dist/` and the Control UI once, upload reusable artifacts for downstream jobs | Node-relevant changes | +| `checks-fast-core` | Fast Linux correctness lanes such as bundled/plugin-contract/protocol checks | Node-relevant changes | +| `checks-node-extensions` | Full bundled-plugin test shards across the extension suite | Node-relevant changes | +| `checks-node-core-test` | Core Node test shards, excluding channel, bundled, contract, and extension lanes | Node-relevant changes | +| `extension-fast` | Focused tests for only the changed bundled plugins | When extension changes are detected | +| `check` | Main local gate in CI: `pnpm check`, `pnpm check:test-types`, and `pnpm build:strict-smoke` | Node-relevant changes | +| `check-additional` | Architecture, boundary, import-cycle guards plus the gateway watch regression harness | Node-relevant changes | +| `build-smoke` | Built-CLI smoke tests and startup-memory smoke | Node-relevant changes | +| `checks` | Remaining Linux Node lanes: channel tests and push-only Node 22 compatibility | Node-relevant changes | +| `check-docs` | Docs formatting, lint, and broken-link checks | Docs changed | +| `skills-python` | Ruff + pytest for Python-backed skills | Python-skill-relevant changes | +| `checks-windows` | Windows-specific test lanes | Windows-relevant changes | +| `macos-node` | macOS TypeScript test lane using the shared built artifacts | macOS-relevant changes | +| `macos-swift` | Swift lint, build, and tests for the macOS app | macOS-relevant changes | +| `android` | Android build and test matrix | Android-relevant changes | ## Fail-Fast Order @@ -57,7 +57,8 @@ On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull ## Local Equivalents ```bash -pnpm check # fast local gate: project-reference tsgo + sharded lint + parallel fast guards +pnpm check # fast local gate: production tsgo + sharded lint + parallel fast guards +pnpm check:test-types pnpm check:timed # same gate with per-stage timings pnpm build:strict-smoke pnpm check:architecture diff --git a/docs/help/testing.md b/docs/help/testing.md index a7260d8163d..230b9c3eb69 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -22,7 +22,7 @@ This doc is a “how we test” guide: Most days: -- Full gate (expected before push): `pnpm build && pnpm check && pnpm test` +- Full gate (expected before push): `pnpm build && pnpm check && pnpm check:test-types && pnpm test` - Faster local full-suite run on a roomy machine: `pnpm test:max` - Direct Vitest watch loop: `pnpm test:watch` - Direct file targeting now routes extension/channel paths too: `pnpm test extensions/discord/src/monitor/message-handler.preflight.test.ts` diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index b498a0f0808..eebe1221ecd 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -37,6 +37,8 @@ OpenClaw has three public release lanes: ## Release preflight +- Run `pnpm check:test-types` before release preflight so test TypeScript stays + covered outside the faster local `pnpm check` gate - Run `pnpm check:architecture` before release preflight so the broader import cycle and architecture boundary checks are green outside the faster local gate - Run `pnpm build && pnpm ui:build` before `pnpm release:check` so the expected diff --git a/docs/reference/test.md b/docs/reference/test.md index cbbcb949afe..829de18c930 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -38,6 +38,7 @@ title: "Tests" For local PR land/gate checks, run: - `pnpm check` +- `pnpm check:test-types` - `pnpm build` - `pnpm test` - `pnpm check:docs` diff --git a/package.json b/package.json index 645898b150b..666d8313730 100644 --- a/package.json +++ b/package.json @@ -1247,7 +1247,9 @@ "check:madge-import-cycles": "node --import tsx scripts/check-madge-import-cycles.ts", "check:no-conflict-markers": "node scripts/check-no-conflict-markers.mjs", "check:static-import-sccs": "pnpm check:madge-import-cycles", + "check:test-types": "pnpm tsgo:test", "check:timed": "node scripts/check-timed.mjs", + "check:timed:all-types": "node scripts/check-timed.mjs --include-test-types", "check:timed:architecture": "node scripts/check-timed.mjs --include-architecture", "codex-app-server:protocol:check": "node --import tsx scripts/check-codex-app-server-protocol.ts", "config:channels:check": "node --import tsx scripts/generate-bundled-channel-config-metadata.ts --check", diff --git a/scripts/build-all.mjs b/scripts/build-all.mjs index f38482b98d2..43a272283e6 100644 --- a/scripts/build-all.mjs +++ b/scripts/build-all.mjs @@ -1,11 +1,15 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; import { pathToFileURL } from "node:url"; import { resolvePnpmRunner } from "./pnpm-runner.mjs"; const nodeBin = process.execPath; const WINDOWS_BUILD_MAX_OLD_SPACE_MB = 4096; +const BUILD_CACHE_VERSION = 2; export const BUILD_ALL_STEPS = [ { label: "canvas:a2ui:bundle", kind: "pnpm", pnpmArgs: ["canvas:a2ui:bundle"] }, { label: "tsdown", kind: "node", args: ["scripts/tsdown-build.mjs"] }, @@ -14,6 +18,16 @@ export const BUILD_ALL_STEPS = [ label: "write-npm-update-compat-sidecars", kind: "node", args: ["--import", "tsx", "scripts/write-npm-update-compat-sidecars.ts"], + cache: { + inputs: [ + "scripts/write-npm-update-compat-sidecars.ts", + "src/infra/npm-update-compat-sidecars.ts", + ], + outputs: [ + "dist/extensions/qa-channel/runtime-api.js", + "dist/extensions/qa-lab/runtime-api.js", + ], + }, }, { label: "build-stamp", kind: "node", args: ["scripts/build-stamp.mjs"] }, { @@ -21,11 +35,30 @@ export const BUILD_ALL_STEPS = [ kind: "pnpm", pnpmArgs: ["build:plugin-sdk:dts"], windowsNodeOptions: `--max-old-space-size=${WINDOWS_BUILD_MAX_OLD_SPACE_MB}`, + cache: { + inputs: [ + "tsconfig.json", + "tsconfig.plugin-sdk.dts.json", + "src/plugin-sdk", + "src/types", + "src/video-generation/dashscope-compatible.ts", + "src/video-generation/types.ts", + ], + outputs: ["dist/plugin-sdk/.tsbuildinfo", "dist/plugin-sdk/src"], + }, }, { label: "write-plugin-sdk-entry-dts", kind: "node", args: ["--import", "tsx", "scripts/write-plugin-sdk-entry-dts.ts"], + cache: { + inputs: [ + "scripts/write-plugin-sdk-entry-dts.ts", + "scripts/lib/plugin-sdk-entrypoints.json", + "dist/plugin-sdk/src/plugin-sdk", + ], + outputs: ["dist/plugin-sdk", "packages/plugin-sdk/dist/src/plugin-sdk"], + }, }, { label: "check-plugin-sdk-exports", @@ -36,16 +69,32 @@ export const BUILD_ALL_STEPS = [ label: "canvas-a2ui-copy", kind: "node", args: ["--import", "tsx", "scripts/canvas-a2ui-copy.ts"], + cache: { + inputs: ["scripts/canvas-a2ui-copy.ts", "src/canvas-host/a2ui"], + outputs: ["dist/canvas-host/a2ui/index.html", "dist/canvas-host/a2ui/a2ui.bundle.js"], + }, }, { label: "copy-hook-metadata", kind: "node", args: ["--import", "tsx", "scripts/copy-hook-metadata.ts"], + cache: { + inputs: ["scripts/copy-hook-metadata.ts", "scripts/lib/copy-assets.ts", "src/hooks/bundled"], + outputs: ["dist/bundled"], + }, }, { label: "copy-export-html-templates", kind: "node", args: ["--import", "tsx", "scripts/copy-export-html-templates.ts"], + cache: { + inputs: [ + "scripts/copy-export-html-templates.ts", + "scripts/lib/copy-assets.ts", + "src/auto-reply/reply/export-html", + ], + outputs: ["dist/export-html"], + }, }, { label: "write-build-info", @@ -142,6 +191,171 @@ export function resolveBuildAllStep(step, params = {}) { }; } +function listFilesRecursively(rootPath, fsImpl) { + let stat; + try { + stat = fsImpl.statSync(rootPath); + } catch { + return []; + } + if (stat.isFile()) { + return [rootPath]; + } + if (!stat.isDirectory()) { + return []; + } + const out = []; + const entries = fsImpl.readdirSync(rootPath, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === ".DS_Store") { + continue; + } + const entryPath = path.join(rootPath, entry.name); + if (entry.isDirectory()) { + out.push(...listFilesRecursively(entryPath, fsImpl)); + } else if (entry.isFile()) { + out.push(entryPath); + } + } + return out; +} + +function listCacheFiles(rootDir, entries, fsImpl) { + return entries + .flatMap((entry) => listFilesRecursively(path.resolve(rootDir, entry), fsImpl)) + .toSorted(); +} + +function resolveCachePaths(rootDir, step) { + const safeLabel = step.label.replace(/[^a-zA-Z0-9._-]+/g, "_"); + const cacheDir = path.resolve(rootDir, ".artifacts/build-all-cache", safeLabel); + return { + cacheDir, + outputRoot: path.join(cacheDir, "outputs"), + stampPath: path.join(cacheDir, "stamp.json"), + }; +} + +function hashInputFiles(rootDir, files, fsImpl) { + const hash = createHash("sha256"); + hash.update(`v${BUILD_CACHE_VERSION}\0`); + for (const file of files) { + hash.update(path.relative(rootDir, file)); + hash.update("\0"); + hash.update(fsImpl.readFileSync(file)); + hash.update("\0"); + } + return hash.digest("hex"); +} + +function readCacheStamp(stampPath, fsImpl) { + try { + return JSON.parse(fsImpl.readFileSync(stampPath, "utf8")); + } catch { + return undefined; + } +} + +function hasAllFiles(rootDir, relativeFiles, fsImpl) { + return relativeFiles.every((relativeFile) => { + try { + return fsImpl.statSync(path.resolve(rootDir, relativeFile)).isFile(); + } catch { + return false; + } + }); +} + +function copyFileSync(fsImpl, sourcePath, targetPath) { + fsImpl.mkdirSync(path.dirname(targetPath), { recursive: true }); + fsImpl.copyFileSync(sourcePath, targetPath); +} + +export function resolveBuildAllStepCacheState(step, params = {}) { + if (!step.cache) { + return { cacheable: false, fresh: false, reason: "no-cache" }; + } + const rootDir = params.rootDir ?? process.cwd(); + const fsImpl = params.fs ?? fs; + const inputFiles = listCacheFiles(rootDir, step.cache.inputs, fsImpl); + if (inputFiles.length === 0) { + return { cacheable: true, fresh: false, reason: "missing-inputs" }; + } + const signature = hashInputFiles(rootDir, inputFiles, fsImpl); + const { outputRoot, stampPath } = resolveCachePaths(rootDir, step); + const stamp = readCacheStamp(stampPath, fsImpl); + const outputFiles = listCacheFiles(rootDir, step.cache.outputs, fsImpl); + const relativeOutputFiles = outputFiles.map((file) => path.relative(rootDir, file)); + const stampedOutputs = Array.isArray(stamp?.outputs) ? stamp.outputs : []; + const stampMatches = stamp?.version === BUILD_CACHE_VERSION && stamp.signature === signature; + const actualOutputsPresent = + stampedOutputs.length > 0 && hasAllFiles(rootDir, stampedOutputs, fsImpl); + const cachedOutputsPresent = + stampedOutputs.length > 0 && hasAllFiles(outputRoot, stampedOutputs, fsImpl); + const restorable = stampMatches && !actualOutputsPresent && cachedOutputsPresent; + const fresh = stampMatches && (actualOutputsPresent || cachedOutputsPresent); + return { + cacheable: true, + fresh, + restorable, + reason: fresh ? (restorable ? "fresh-cache" : "fresh") : "stale", + signature, + outputRoot, + stampPath, + inputFiles: inputFiles.length, + outputFiles: outputFiles.length, + relativeOutputFiles, + stampedOutputs, + }; +} + +export function writeBuildAllStepCacheStamp(step, cacheState, params = {}) { + if ( + !cacheState.cacheable || + !cacheState.signature || + !cacheState.stampPath || + !cacheState.outputRoot || + !cacheState.relativeOutputFiles?.length + ) { + return; + } + const fsImpl = params.fs ?? fs; + const rootDir = params.rootDir ?? process.cwd(); + for (const relativeFile of cacheState.relativeOutputFiles) { + copyFileSync( + fsImpl, + path.resolve(rootDir, relativeFile), + path.resolve(cacheState.outputRoot, relativeFile), + ); + } + fsImpl.mkdirSync(path.dirname(cacheState.stampPath), { recursive: true }); + fsImpl.writeFileSync( + cacheState.stampPath, + `${JSON.stringify({ + version: BUILD_CACHE_VERSION, + label: step.label, + signature: cacheState.signature, + outputs: cacheState.relativeOutputFiles, + })}\n`, + ); +} + +export function restoreBuildAllStepCacheOutputs(cacheState, params = {}) { + if (!cacheState.restorable || !cacheState.outputRoot || !cacheState.stampedOutputs?.length) { + return false; + } + const fsImpl = params.fs ?? fs; + const rootDir = params.rootDir ?? process.cwd(); + for (const relativeFile of cacheState.stampedOutputs) { + copyFileSync( + fsImpl, + path.resolve(cacheState.outputRoot, relativeFile), + path.resolve(rootDir, relativeFile), + ); + } + return true; +} + function isMainModule() { const argv1 = process.argv[1]; if (!argv1) { @@ -153,6 +367,12 @@ function isMainModule() { if (isMainModule()) { const profile = process.argv[2] ?? "full"; for (const step of resolveBuildAllSteps(profile)) { + const cacheState = resolveBuildAllStepCacheState(step); + if (process.env.OPENCLAW_BUILD_CACHE !== "0" && cacheState.fresh) { + restoreBuildAllStepCacheOutputs(cacheState); + console.error(`[build-all] ${step.label} (cached)`); + continue; + } console.error(`[build-all] ${step.label}`); const invocation = resolveBuildAllStep(step); const result = spawnSync(invocation.command, invocation.args, invocation.options); @@ -160,6 +380,7 @@ if (isMainModule()) { if (result.status !== 0) { process.exit(result.status); } + writeBuildAllStepCacheStamp(step, resolveBuildAllStepCacheState(step)); continue; } process.exit(1); diff --git a/scripts/check.mjs b/scripts/check.mjs index c36292b2234..007f4be6fa8 100644 --- a/scripts/check.mjs +++ b/scripts/check.mjs @@ -4,6 +4,7 @@ import { performance } from "node:perf_hooks"; export async function main(argv = process.argv.slice(2)) { const timed = argv.includes("--timed"); const includeArchitecture = argv.includes("--include-architecture"); + const includeTestTypes = argv.includes("--include-test-types"); const tailChecks = [ { name: "webhook body guard", args: ["lint:webhook:no-low-level-body-read"] }, @@ -17,7 +18,7 @@ export async function main(argv = process.argv.slice(2)) { const stages = [ { name: "preflight guards", - parallel: false, + parallel: true, commands: [ { name: "conflict markers", args: ["check:no-conflict-markers"] }, { name: "tool display", args: ["tool-display:check"] }, @@ -27,7 +28,12 @@ export async function main(argv = process.argv.slice(2)) { { name: "typecheck", parallel: false, - commands: [{ name: "typecheck", args: ["tsgo:all"] }], + commands: [ + { + name: includeTestTypes ? "typecheck all" : "typecheck prod", + args: [includeTestTypes ? "tsgo:all" : "tsgo:prod"], + }, + ], }, { name: "lint", diff --git a/src/auto-reply/reply/directive-handling.model.test.ts b/src/auto-reply/reply/directive-handling.model.test.ts index 3474b3654f7..463b3c51985 100644 --- a/src/auto-reply/reply/directive-handling.model.test.ts +++ b/src/auto-reply/reply/directive-handling.model.test.ts @@ -24,17 +24,35 @@ vi.mock("../../agents/auth-profiles.js", () => ({ resolveAuthStorePathForDisplay: () => "/tmp/auth-profiles.json", })); -vi.mock("../../agents/auth-profiles/store.js", () => ({ - ensureAuthProfileStore: () => ({ +vi.mock("../../agents/auth-profiles/store.js", () => { + const store = () => ({ version: 1, profiles: authProfilesStoreMock.profiles, - }), - findPersistedAuthProfileCredential: ({ profileId }: { profileId: string }) => - authProfilesStoreMock.profiles[profileId], - hasAnyAuthProfileStoreSource: () => Object.keys(authProfilesStoreMock.profiles).length > 0, - saveAuthProfileStore: vi.fn(), - updateAuthProfileStoreWithLock: vi.fn(), -})); + }); + return { + clearRuntimeAuthProfileStoreSnapshots: () => { + authProfilesStoreMock.profiles = {}; + }, + ensureAuthProfileStore: store, + ensureAuthProfileStoreForLocalUpdate: store, + findPersistedAuthProfileCredential: ({ profileId }: { profileId: string }) => + authProfilesStoreMock.profiles[profileId], + hasAnyAuthProfileStoreSource: () => Object.keys(authProfilesStoreMock.profiles).length > 0, + loadAuthProfileStore: store, + loadAuthProfileStoreForRuntime: store, + loadAuthProfileStoreForSecretsRuntime: store, + loadAuthProfileStoreWithoutExternalProfiles: store, + replaceRuntimeAuthProfileStoreSnapshots: ( + snapshots: Array<{ + store?: { profiles?: Record }; + }>, + ) => { + authProfilesStoreMock.profiles = snapshots[0]?.store?.profiles ?? {}; + }, + saveAuthProfileStore: vi.fn(), + updateAuthProfileStoreWithLock: vi.fn(async ({ update }) => update(store())), + }; +}); import { resolveAgentDir, resolveSessionAgentId } from "../../agents/agent-scope.js"; import { diff --git a/test/helpers/channels/imessage-test-plugin.ts b/test/helpers/channels/imessage-test-plugin.ts index 1115717d55c..62861aefd40 100644 --- a/test/helpers/channels/imessage-test-plugin.ts +++ b/test/helpers/channels/imessage-test-plugin.ts @@ -1,5 +1,5 @@ import type { ChannelOutboundAdapter } from "openclaw/plugin-sdk/channel-contract"; -import type { ChannelPlugin } from "openclaw/plugin-sdk/core"; +import type { ChannelPlugin } from "openclaw/plugin-sdk/channel-plugin-common"; import { loadBundledPluginTestApiSync } from "../../../src/test-utils/bundled-plugin-public-surface.js"; type CreateIMessageTestPlugin = (params?: { outbound?: ChannelOutboundAdapter }) => ChannelPlugin; diff --git a/test/helpers/plugins/plugin-runtime-mock.ts b/test/helpers/plugins/plugin-runtime-mock.ts index cbe276154bd..a9d20d8efb5 100644 --- a/test/helpers/plugins/plugin-runtime-mock.ts +++ b/test/helpers/plugins/plugin-runtime-mock.ts @@ -1,4 +1,3 @@ -import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "openclaw/plugin-sdk/agent-runtime"; import { vi } from "vitest"; import { removeAckReactionAfterReply, @@ -10,6 +9,9 @@ import { } from "../../../src/channels/mention-gating.js"; import type { PluginRuntime } from "../../../src/plugins/runtime/types.js"; +const DEFAULT_PROVIDER = "openai"; +const DEFAULT_MODEL = "gpt-5.4"; + type DeepPartial = { [K in keyof T]?: T[K] extends (...args: never[]) => unknown ? T[K] diff --git a/test/scripts/build-all.test.ts b/test/scripts/build-all.test.ts index 0ba92b5129c..3d013cb513d 100644 --- a/test/scripts/build-all.test.ts +++ b/test/scripts/build-all.test.ts @@ -1,9 +1,15 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; import { BUILD_ALL_PROFILES, BUILD_ALL_STEPS, + resolveBuildAllStepCacheState, resolveBuildAllStep, resolveBuildAllSteps, + restoreBuildAllStepCacheOutputs, + writeBuildAllStepCacheStamp, } from "../../scripts/build-all.mjs"; describe("resolveBuildAllStep", () => { @@ -102,3 +108,100 @@ describe("resolveBuildAllSteps", () => { expect(() => resolveBuildAllSteps("wat")).toThrow("Unknown build profile: wat"); }); }); + +describe("resolveBuildAllStepCacheState", () => { + it("marks cacheable steps fresh when the input signature matches", () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-")); + try { + const inputPath = path.join(rootDir, "src/input.ts"); + const outputPath = path.join(rootDir, "dist/output.js"); + fs.mkdirSync(path.dirname(inputPath), { recursive: true }); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(inputPath, "input"); + fs.writeFileSync(outputPath, "output"); + const step = { + label: "cached", + cache: { + inputs: ["src"], + outputs: ["dist"], + }, + }; + const cacheState = resolveBuildAllStepCacheState(step, { rootDir }); + writeBuildAllStepCacheStamp(step, cacheState, { rootDir }); + + expect(resolveBuildAllStepCacheState(step, { rootDir })).toMatchObject({ + cacheable: true, + fresh: true, + reason: "fresh", + inputFiles: 1, + outputFiles: 1, + }); + } finally { + fs.rmSync(rootDir, { force: true, recursive: true }); + } + }); + + it("marks cacheable steps stale when an input changes", () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-")); + try { + const inputPath = path.join(rootDir, "src/input.ts"); + const outputPath = path.join(rootDir, "dist/output.js"); + fs.mkdirSync(path.dirname(inputPath), { recursive: true }); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(inputPath, "input"); + fs.writeFileSync(outputPath, "output"); + const step = { + label: "cached", + cache: { + inputs: ["src"], + outputs: ["dist"], + }, + }; + const cacheState = resolveBuildAllStepCacheState(step, { rootDir }); + writeBuildAllStepCacheStamp(step, cacheState, { rootDir }); + fs.writeFileSync(inputPath, "changed"); + + expect(resolveBuildAllStepCacheState(step, { rootDir })).toMatchObject({ + cacheable: true, + fresh: false, + reason: "stale", + }); + } finally { + fs.rmSync(rootDir, { force: true, recursive: true }); + } + }); + + it("restores cached outputs when generated files were removed", () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-")); + try { + const inputPath = path.join(rootDir, "src/input.ts"); + const outputPath = path.join(rootDir, "dist/output.js"); + fs.mkdirSync(path.dirname(inputPath), { recursive: true }); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(inputPath, "input"); + fs.writeFileSync(outputPath, "output"); + const step = { + label: "cached", + cache: { + inputs: ["src"], + outputs: ["dist"], + }, + }; + const cacheState = resolveBuildAllStepCacheState(step, { rootDir }); + writeBuildAllStepCacheStamp(step, cacheState, { rootDir }); + fs.rmSync(path.join(rootDir, "dist"), { force: true, recursive: true }); + + const restorable = resolveBuildAllStepCacheState(step, { rootDir }); + expect(restorable).toMatchObject({ + cacheable: true, + fresh: true, + reason: "fresh-cache", + restorable: true, + }); + expect(restoreBuildAllStepCacheOutputs(restorable, { rootDir })).toBe(true); + expect(fs.readFileSync(outputPath, "utf8")).toBe("output"); + } finally { + fs.rmSync(rootDir, { force: true, recursive: true }); + } + }); +});