test: add Crestodian QA lab setup scenario

This commit is contained in:
Peter Steinberger
2026-04-25 13:15:05 +01:00
parent c977643460
commit b26367e22f
6 changed files with 349 additions and 5 deletions

View File

@@ -306,7 +306,12 @@ pnpm test:docker:crestodian-first-run
That lane starts with an empty state dir, routes bare `openclaw` to Crestodian,
sets the default model, creates an additional agent, configures Discord through
a token SecretRef, validates config, and checks the audit log.
a plugin enablement plus token SecretRef, validates config, and checks the audit
log. QA Lab also has a repo-backed scenario for the same Ring 0 flow:
```bash
pnpm openclaw qa suite --scenario crestodian-ring-zero-setup
```
## Related

View File

@@ -65,8 +65,10 @@ When debugging real providers/models (requires real creds):
config write.
- Crestodian first-run Docker smoke: `pnpm test:docker:crestodian-first-run`
- Starts from an empty OpenClaw state dir, routes bare `openclaw` to
Crestodian, applies setup/model/agent/Discord SecretRef writes, validates
config, and verifies audit entries.
Crestodian, applies setup/model/agent/Discord plugin + SecretRef writes,
validates config, and verifies audit entries. The same Ring 0 setup path is
also covered in QA Lab by
`pnpm openclaw qa suite --scenario crestodian-ring-zero-setup`.
- Moonshot/Kimi cost smoke: with `MOONSHOT_API_KEY` set, run
`openclaw models list --provider moonshot --json`, then run an isolated
`openclaw agent --local --session-id live-kimi-cost --message 'Reply exactly: KIMI_LIVE_OK' --thinking off --json`

View File

@@ -103,6 +103,48 @@ describe("qa suite runtime agent process helpers", () => {
);
});
it("merges isolated env overrides into qa cli runs", async () => {
const child = createSpawnedProcess();
spawnMock.mockReturnValue(child);
const pending = runQaCli(
{
repoRoot: "/repo",
gateway: {
tempRoot: "/tmp/runtime",
runtimeEnv: { PATH: "/usr/bin", OPENCLAW_STATE_DIR: "/tmp/default-state" },
},
primaryModel: "openai/gpt-5.4",
alternateModel: "openai/gpt-5.4-mini",
providerMode: "mock-openai",
} as never,
["crestodian", "-m", "overview"],
{
env: {
OPENCLAW_STATE_DIR: "/tmp/isolated-state",
OPENCLAW_CONFIG_PATH: "/tmp/isolated-state/openclaw.json",
},
},
);
await waitForSpawnCount(1);
child.stdout.emit("data", Buffer.from("ok\n"));
child.emit("exit", 0);
await expect(pending).resolves.toBe("ok");
expect(spawnMock).toHaveBeenCalledWith(
"/usr/bin/node",
["/repo/dist/index.js", "crestodian", "-m", "overview"],
expect.objectContaining({
env: expect.objectContaining({
PATH: "/usr/bin",
OPENCLAW_STATE_DIR: "/tmp/isolated-state",
OPENCLAW_CONFIG_PATH: "/tmp/isolated-state/openclaw.json",
}),
}),
);
});
it("parses json qa cli output when requested", async () => {
const child = createSpawnedProcess();
spawnMock.mockReturnValue(child);

View File

@@ -73,7 +73,7 @@ async function runQaCli(
"gateway" | "repoRoot" | "primaryModel" | "alternateModel" | "providerMode"
>,
args: string[],
opts?: { timeoutMs?: number; json?: boolean },
opts?: { timeoutMs?: number; json?: boolean; env?: NodeJS.ProcessEnv },
) {
const stdout: Buffer[] = [];
const stderr: Buffer[] = [];
@@ -82,7 +82,10 @@ async function runQaCli(
await new Promise<void>((resolve, reject) => {
const child = spawn(nodeExecPath, [distEntryPath, ...args], {
cwd: env.gateway.tempRoot,
env: env.gateway.runtimeEnv,
env: {
...env.gateway.runtimeEnv,
...opts?.env,
},
stdio: ["ignore", "pipe", "pipe"],
});
const timeout = setTimeout(() => {

View File

@@ -0,0 +1,257 @@
# Crestodian ring-zero setup
```yaml qa-scenario
id: crestodian-ring-zero-setup
title: Crestodian ring-zero setup
surface: config
coverage:
primary:
- config.crestodian-setup
secondary:
- channels.discord-config
- agents.create
objective: Verify Crestodian can bootstrap a fresh OpenClaw config, set the default model, create an agent, configure Discord through a SecretRef, validate config, and leave an audit trail.
successCriteria:
- Crestodian reports missing config in an empty state dir.
- Crestodian setup writes a workspace and default model.
- Crestodian creates a non-main agent with its own workspace and model.
- Crestodian enables the Discord plugin before writing Discord channel config.
- Crestodian configures Discord through an env SecretRef without persisting the raw token.
- Config validation passes and audit entries exist for every applied write.
docsRefs:
- docs/cli/crestodian.md
- docs/channels/discord.md
- docs/help/testing.md
codeRefs:
- src/crestodian/operations.ts
- scripts/e2e/crestodian-first-run-docker-client.ts
- extensions/qa-lab/src/suite-runtime-agent-process.ts
execution:
kind: flow
summary: Drive the public Crestodian CLI in an isolated fresh state dir and verify setup/model/agent/Discord/audit results.
config:
stateDirName: crestodian-ring-zero-state
defaultWorkspaceName: crestodian-main-workspace
agentWorkspaceName: crestodian-reef-workspace
agentId: reef
model: openai/gpt-5.2
discordEnv: DISCORD_BOT_TOKEN
discordToken: openclaw-crestodian-qa-discord-token
```
```yaml qa-flow
steps:
- name: bootstraps config through Crestodian CLI
actions:
- set: stateDir
value:
expr: "path.join(env.gateway.tempRoot, config.stateDirName)"
- set: configPath
value:
expr: "path.join(stateDir, 'openclaw.json')"
- set: defaultWorkspace
value:
expr: "path.join(env.gateway.tempRoot, config.defaultWorkspaceName)"
- set: agentWorkspace
value:
expr: "path.join(env.gateway.tempRoot, config.agentWorkspaceName)"
- set: crestodianEnv
value:
expr: "({ OPENCLAW_STATE_DIR: stateDir, OPENCLAW_CONFIG_PATH: configPath, OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(env.repoRoot, 'dist', 'extensions'), [config.discordEnv]: config.discordToken })"
- call: fs.rm
args:
- ref: stateDir
- recursive: true
force: true
- call: fs.mkdir
args:
- ref: stateDir
- recursive: true
- call: runQaCli
saveAs: overviewOutput
args:
- ref: env
- - crestodian
- -m
- overview
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(overviewOutput).includes('Config: missing')"
message:
expr: "`fresh Crestodian overview did not report missing config: ${overviewOutput}`"
- assert:
expr: 'String(overviewOutput).includes(''Next: run "setup" to create a starter config'')'
message:
expr: "`fresh Crestodian overview did not recommend setup: ${overviewOutput}`"
- call: runQaCli
saveAs: setupOutput
args:
- ref: env
- - crestodian
- --yes
- -m
- expr: "`setup workspace ${defaultWorkspace} model ${config.model}`"
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(setupOutput).includes('[crestodian] done: crestodian.setup')"
message:
expr: "`Crestodian setup did not apply: ${setupOutput}`"
- call: runQaCli
saveAs: modelOutput
args:
- ref: env
- - crestodian
- --yes
- -m
- expr: "`set default model ${config.model}`"
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(modelOutput).includes('[crestodian] done: config.setDefaultModel')"
message:
expr: "`Crestodian model update did not apply: ${modelOutput}`"
- call: runQaCli
saveAs: agentOutput
args:
- ref: env
- - crestodian
- --yes
- -m
- expr: "`create agent ${config.agentId} workspace ${agentWorkspace} model ${config.model}`"
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(agentOutput).includes('[crestodian] done: agents.create')"
message:
expr: "`Crestodian agent creation did not apply: ${agentOutput}`"
- call: runQaCli
saveAs: discordPluginAllowOutput
args:
- ref: env
- - crestodian
- --yes
- -m
- config set plugins.allow ["discord"]
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(discordPluginAllowOutput).includes('[crestodian] done: config.set')"
message:
expr: "`Crestodian Discord plugin allowlist did not apply: ${discordPluginAllowOutput}`"
- call: runQaCli
saveAs: discordPluginEntryOutput
args:
- ref: env
- - crestodian
- --yes
- -m
- config set plugins.entries.discord.enabled true
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(discordPluginEntryOutput).includes('[crestodian] done: config.set')"
message:
expr: "`Crestodian Discord plugin entry did not apply: ${discordPluginEntryOutput}`"
- call: runQaCli
saveAs: discordTokenOutput
args:
- ref: env
- - crestodian
- --yes
- -m
- expr: "`config set-ref channels.discord.token env ${config.discordEnv}`"
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(discordTokenOutput).includes('[crestodian] done: config.setRef')"
message:
expr: "`Crestodian Discord SecretRef did not apply: ${discordTokenOutput}`"
- call: runQaCli
saveAs: discordEnabledOutput
args:
- ref: env
- - crestodian
- --yes
- -m
- config set channels.discord.enabled true
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(discordEnabledOutput).includes('[crestodian] done: config.set')"
message:
expr: "`Crestodian Discord enable did not apply: ${discordEnabledOutput}`"
- call: runQaCli
saveAs: validationOutput
args:
- ref: env
- - crestodian
- -m
- validate config
- timeoutMs: 60000
env:
ref: crestodianEnv
- assert:
expr: "String(validationOutput).includes('Config valid:')"
message:
expr: "`Crestodian config validation did not pass: ${validationOutput}`"
- set: writtenConfig
value:
expr: "JSON.parse(await fs.readFile(configPath, 'utf8'))"
- set: agent
value:
expr: "writtenConfig.agents?.list?.find((candidate) => candidate.id === config.agentId)"
- assert:
expr: "writtenConfig.agents?.defaults?.workspace === defaultWorkspace"
message:
expr: "`default workspace mismatch: ${JSON.stringify(writtenConfig.agents?.defaults)}`"
- assert:
expr: "writtenConfig.agents?.defaults?.model?.primary === config.model"
message:
expr: "`default model mismatch: ${JSON.stringify(writtenConfig.agents?.defaults?.model)}`"
- assert:
expr: "agent?.workspace === agentWorkspace && agent?.model === config.model"
message:
expr: "`agent config mismatch: ${JSON.stringify(agent)}`"
- assert:
expr: "writtenConfig.plugins?.allow?.includes('discord') && writtenConfig.plugins?.entries?.discord?.enabled === true"
message:
expr: "`Discord plugin was not enabled: ${JSON.stringify(writtenConfig.plugins)}`"
- assert:
expr: "writtenConfig.channels?.discord?.enabled === true"
message:
expr: "`Discord was not enabled: ${JSON.stringify(writtenConfig.channels?.discord)}`"
- assert:
expr: "writtenConfig.channels?.discord?.token?.source === 'env' && writtenConfig.channels?.discord?.token?.id === config.discordEnv"
message:
expr: "`Discord token was not an env SecretRef: ${JSON.stringify(writtenConfig.channels?.discord?.token)}`"
- assert:
expr: "!JSON.stringify(writtenConfig.channels?.discord ?? {}).includes(config.discordToken)"
message: Crestodian persisted the raw Discord token.
- set: auditText
value:
expr: "await fs.readFile(path.join(stateDir, 'audit', 'crestodian.jsonl'), 'utf8')"
- forEach:
items:
- crestodian.setup
- config.setDefaultModel
- agents.create
- config.setRef
- config.set
item: operation
actions:
- assert:
expr: 'auditText.includes(`"operation":"${operation}"`)'
message:
expr: "`missing audit entry for ${operation}: ${auditText}`"
detailsExpr: "`stateDir=${stateDir}\\nconfigPath=${configPath}\\nagent=${JSON.stringify(agent)}\\nDiscord SecretRef=${JSON.stringify(writtenConfig.channels?.discord?.token)}`"
```

View File

@@ -108,6 +108,36 @@ async function main() {
"Crestodian agent creation did not apply",
);
clearConfigCache();
const discordPluginAllowRuntime = createRuntime();
await runCrestodian(
{
message: 'config set plugins.allow ["discord"]',
yes: true,
interactive: false,
},
discordPluginAllowRuntime.runtime,
);
assert(
discordPluginAllowRuntime.lines.join("\n").includes("[crestodian] done: config.set"),
"Crestodian Discord plugin allowlist did not apply",
);
clearConfigCache();
const discordPluginEntryRuntime = createRuntime();
await runCrestodian(
{
message: "config set plugins.entries.discord.enabled true",
yes: true,
interactive: false,
},
discordPluginEntryRuntime.runtime,
);
assert(
discordPluginEntryRuntime.lines.join("\n").includes("[crestodian] done: config.set"),
"Crestodian Discord plugin entry did not apply",
);
clearConfigCache();
const discordTokenRuntime = createRuntime();
await runCrestodian(
@@ -162,6 +192,11 @@ async function main() {
assert(reef, "Crestodian did not create reef agent");
assert(reef.workspace === "/tmp/openclaw-reef", "Crestodian did not write reef workspace");
assert(reef.model === "openai/gpt-5.2", "Crestodian did not write reef model");
assert(config.plugins?.allow?.includes("discord"), "Crestodian did not allow Discord plugin");
assert(
config.plugins?.entries?.discord?.enabled === true,
"Crestodian did not enable Discord plugin entry",
);
assert(config.channels?.discord?.enabled === true, "Crestodian did not enable Discord");
const discordToken = config.channels?.discord?.token;
assert(