mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 06:00:43 +00:00
test: add Crestodian QA lab setup scenario
This commit is contained in:
@@ -306,7 +306,12 @@ pnpm test:docker:crestodian-first-run
|
||||
|
||||
That lane starts with an empty state dir, routes bare `openclaw` to Crestodian,
|
||||
sets the default model, creates an additional agent, configures Discord through
|
||||
a token SecretRef, validates config, and checks the audit log.
|
||||
a plugin enablement plus token SecretRef, validates config, and checks the audit
|
||||
log. QA Lab also has a repo-backed scenario for the same Ring 0 flow:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa suite --scenario crestodian-ring-zero-setup
|
||||
```
|
||||
|
||||
## Related
|
||||
|
||||
|
||||
@@ -65,8 +65,10 @@ When debugging real providers/models (requires real creds):
|
||||
config write.
|
||||
- Crestodian first-run Docker smoke: `pnpm test:docker:crestodian-first-run`
|
||||
- Starts from an empty OpenClaw state dir, routes bare `openclaw` to
|
||||
Crestodian, applies setup/model/agent/Discord SecretRef writes, validates
|
||||
config, and verifies audit entries.
|
||||
Crestodian, applies setup/model/agent/Discord plugin + SecretRef writes,
|
||||
validates config, and verifies audit entries. The same Ring 0 setup path is
|
||||
also covered in QA Lab by
|
||||
`pnpm openclaw qa suite --scenario crestodian-ring-zero-setup`.
|
||||
- Moonshot/Kimi cost smoke: with `MOONSHOT_API_KEY` set, run
|
||||
`openclaw models list --provider moonshot --json`, then run an isolated
|
||||
`openclaw agent --local --session-id live-kimi-cost --message 'Reply exactly: KIMI_LIVE_OK' --thinking off --json`
|
||||
|
||||
@@ -103,6 +103,48 @@ describe("qa suite runtime agent process helpers", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("merges isolated env overrides into qa cli runs", async () => {
|
||||
const child = createSpawnedProcess();
|
||||
spawnMock.mockReturnValue(child);
|
||||
|
||||
const pending = runQaCli(
|
||||
{
|
||||
repoRoot: "/repo",
|
||||
gateway: {
|
||||
tempRoot: "/tmp/runtime",
|
||||
runtimeEnv: { PATH: "/usr/bin", OPENCLAW_STATE_DIR: "/tmp/default-state" },
|
||||
},
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4-mini",
|
||||
providerMode: "mock-openai",
|
||||
} as never,
|
||||
["crestodian", "-m", "overview"],
|
||||
{
|
||||
env: {
|
||||
OPENCLAW_STATE_DIR: "/tmp/isolated-state",
|
||||
OPENCLAW_CONFIG_PATH: "/tmp/isolated-state/openclaw.json",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await waitForSpawnCount(1);
|
||||
child.stdout.emit("data", Buffer.from("ok\n"));
|
||||
child.emit("exit", 0);
|
||||
|
||||
await expect(pending).resolves.toBe("ok");
|
||||
expect(spawnMock).toHaveBeenCalledWith(
|
||||
"/usr/bin/node",
|
||||
["/repo/dist/index.js", "crestodian", "-m", "overview"],
|
||||
expect.objectContaining({
|
||||
env: expect.objectContaining({
|
||||
PATH: "/usr/bin",
|
||||
OPENCLAW_STATE_DIR: "/tmp/isolated-state",
|
||||
OPENCLAW_CONFIG_PATH: "/tmp/isolated-state/openclaw.json",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("parses json qa cli output when requested", async () => {
|
||||
const child = createSpawnedProcess();
|
||||
spawnMock.mockReturnValue(child);
|
||||
|
||||
@@ -73,7 +73,7 @@ async function runQaCli(
|
||||
"gateway" | "repoRoot" | "primaryModel" | "alternateModel" | "providerMode"
|
||||
>,
|
||||
args: string[],
|
||||
opts?: { timeoutMs?: number; json?: boolean },
|
||||
opts?: { timeoutMs?: number; json?: boolean; env?: NodeJS.ProcessEnv },
|
||||
) {
|
||||
const stdout: Buffer[] = [];
|
||||
const stderr: Buffer[] = [];
|
||||
@@ -82,7 +82,10 @@ async function runQaCli(
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(nodeExecPath, [distEntryPath, ...args], {
|
||||
cwd: env.gateway.tempRoot,
|
||||
env: env.gateway.runtimeEnv,
|
||||
env: {
|
||||
...env.gateway.runtimeEnv,
|
||||
...opts?.env,
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const timeout = setTimeout(() => {
|
||||
|
||||
257
qa/scenarios/config/crestodian-ring-zero-setup.md
Normal file
257
qa/scenarios/config/crestodian-ring-zero-setup.md
Normal file
@@ -0,0 +1,257 @@
|
||||
# Crestodian ring-zero setup
|
||||
|
||||
```yaml qa-scenario
|
||||
id: crestodian-ring-zero-setup
|
||||
title: Crestodian ring-zero setup
|
||||
surface: config
|
||||
coverage:
|
||||
primary:
|
||||
- config.crestodian-setup
|
||||
secondary:
|
||||
- channels.discord-config
|
||||
- agents.create
|
||||
objective: Verify Crestodian can bootstrap a fresh OpenClaw config, set the default model, create an agent, configure Discord through a SecretRef, validate config, and leave an audit trail.
|
||||
successCriteria:
|
||||
- Crestodian reports missing config in an empty state dir.
|
||||
- Crestodian setup writes a workspace and default model.
|
||||
- Crestodian creates a non-main agent with its own workspace and model.
|
||||
- Crestodian enables the Discord plugin before writing Discord channel config.
|
||||
- Crestodian configures Discord through an env SecretRef without persisting the raw token.
|
||||
- Config validation passes and audit entries exist for every applied write.
|
||||
docsRefs:
|
||||
- docs/cli/crestodian.md
|
||||
- docs/channels/discord.md
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- src/crestodian/operations.ts
|
||||
- scripts/e2e/crestodian-first-run-docker-client.ts
|
||||
- extensions/qa-lab/src/suite-runtime-agent-process.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Drive the public Crestodian CLI in an isolated fresh state dir and verify setup/model/agent/Discord/audit results.
|
||||
config:
|
||||
stateDirName: crestodian-ring-zero-state
|
||||
defaultWorkspaceName: crestodian-main-workspace
|
||||
agentWorkspaceName: crestodian-reef-workspace
|
||||
agentId: reef
|
||||
model: openai/gpt-5.2
|
||||
discordEnv: DISCORD_BOT_TOKEN
|
||||
discordToken: openclaw-crestodian-qa-discord-token
|
||||
```
|
||||
|
||||
```yaml qa-flow
|
||||
steps:
|
||||
- name: bootstraps config through Crestodian CLI
|
||||
actions:
|
||||
- set: stateDir
|
||||
value:
|
||||
expr: "path.join(env.gateway.tempRoot, config.stateDirName)"
|
||||
- set: configPath
|
||||
value:
|
||||
expr: "path.join(stateDir, 'openclaw.json')"
|
||||
- set: defaultWorkspace
|
||||
value:
|
||||
expr: "path.join(env.gateway.tempRoot, config.defaultWorkspaceName)"
|
||||
- set: agentWorkspace
|
||||
value:
|
||||
expr: "path.join(env.gateway.tempRoot, config.agentWorkspaceName)"
|
||||
- set: crestodianEnv
|
||||
value:
|
||||
expr: "({ OPENCLAW_STATE_DIR: stateDir, OPENCLAW_CONFIG_PATH: configPath, OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(env.repoRoot, 'dist', 'extensions'), [config.discordEnv]: config.discordToken })"
|
||||
- call: fs.rm
|
||||
args:
|
||||
- ref: stateDir
|
||||
- recursive: true
|
||||
force: true
|
||||
- call: fs.mkdir
|
||||
args:
|
||||
- ref: stateDir
|
||||
- recursive: true
|
||||
- call: runQaCli
|
||||
saveAs: overviewOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- -m
|
||||
- overview
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(overviewOutput).includes('Config: missing')"
|
||||
message:
|
||||
expr: "`fresh Crestodian overview did not report missing config: ${overviewOutput}`"
|
||||
- assert:
|
||||
expr: 'String(overviewOutput).includes(''Next: run "setup" to create a starter config'')'
|
||||
message:
|
||||
expr: "`fresh Crestodian overview did not recommend setup: ${overviewOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: setupOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- --yes
|
||||
- -m
|
||||
- expr: "`setup workspace ${defaultWorkspace} model ${config.model}`"
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(setupOutput).includes('[crestodian] done: crestodian.setup')"
|
||||
message:
|
||||
expr: "`Crestodian setup did not apply: ${setupOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: modelOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- --yes
|
||||
- -m
|
||||
- expr: "`set default model ${config.model}`"
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(modelOutput).includes('[crestodian] done: config.setDefaultModel')"
|
||||
message:
|
||||
expr: "`Crestodian model update did not apply: ${modelOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: agentOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- --yes
|
||||
- -m
|
||||
- expr: "`create agent ${config.agentId} workspace ${agentWorkspace} model ${config.model}`"
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(agentOutput).includes('[crestodian] done: agents.create')"
|
||||
message:
|
||||
expr: "`Crestodian agent creation did not apply: ${agentOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: discordPluginAllowOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- --yes
|
||||
- -m
|
||||
- config set plugins.allow ["discord"]
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(discordPluginAllowOutput).includes('[crestodian] done: config.set')"
|
||||
message:
|
||||
expr: "`Crestodian Discord plugin allowlist did not apply: ${discordPluginAllowOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: discordPluginEntryOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- --yes
|
||||
- -m
|
||||
- config set plugins.entries.discord.enabled true
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(discordPluginEntryOutput).includes('[crestodian] done: config.set')"
|
||||
message:
|
||||
expr: "`Crestodian Discord plugin entry did not apply: ${discordPluginEntryOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: discordTokenOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- --yes
|
||||
- -m
|
||||
- expr: "`config set-ref channels.discord.token env ${config.discordEnv}`"
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(discordTokenOutput).includes('[crestodian] done: config.setRef')"
|
||||
message:
|
||||
expr: "`Crestodian Discord SecretRef did not apply: ${discordTokenOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: discordEnabledOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- --yes
|
||||
- -m
|
||||
- config set channels.discord.enabled true
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(discordEnabledOutput).includes('[crestodian] done: config.set')"
|
||||
message:
|
||||
expr: "`Crestodian Discord enable did not apply: ${discordEnabledOutput}`"
|
||||
- call: runQaCli
|
||||
saveAs: validationOutput
|
||||
args:
|
||||
- ref: env
|
||||
- - crestodian
|
||||
- -m
|
||||
- validate config
|
||||
- timeoutMs: 60000
|
||||
env:
|
||||
ref: crestodianEnv
|
||||
- assert:
|
||||
expr: "String(validationOutput).includes('Config valid:')"
|
||||
message:
|
||||
expr: "`Crestodian config validation did not pass: ${validationOutput}`"
|
||||
- set: writtenConfig
|
||||
value:
|
||||
expr: "JSON.parse(await fs.readFile(configPath, 'utf8'))"
|
||||
- set: agent
|
||||
value:
|
||||
expr: "writtenConfig.agents?.list?.find((candidate) => candidate.id === config.agentId)"
|
||||
- assert:
|
||||
expr: "writtenConfig.agents?.defaults?.workspace === defaultWorkspace"
|
||||
message:
|
||||
expr: "`default workspace mismatch: ${JSON.stringify(writtenConfig.agents?.defaults)}`"
|
||||
- assert:
|
||||
expr: "writtenConfig.agents?.defaults?.model?.primary === config.model"
|
||||
message:
|
||||
expr: "`default model mismatch: ${JSON.stringify(writtenConfig.agents?.defaults?.model)}`"
|
||||
- assert:
|
||||
expr: "agent?.workspace === agentWorkspace && agent?.model === config.model"
|
||||
message:
|
||||
expr: "`agent config mismatch: ${JSON.stringify(agent)}`"
|
||||
- assert:
|
||||
expr: "writtenConfig.plugins?.allow?.includes('discord') && writtenConfig.plugins?.entries?.discord?.enabled === true"
|
||||
message:
|
||||
expr: "`Discord plugin was not enabled: ${JSON.stringify(writtenConfig.plugins)}`"
|
||||
- assert:
|
||||
expr: "writtenConfig.channels?.discord?.enabled === true"
|
||||
message:
|
||||
expr: "`Discord was not enabled: ${JSON.stringify(writtenConfig.channels?.discord)}`"
|
||||
- assert:
|
||||
expr: "writtenConfig.channels?.discord?.token?.source === 'env' && writtenConfig.channels?.discord?.token?.id === config.discordEnv"
|
||||
message:
|
||||
expr: "`Discord token was not an env SecretRef: ${JSON.stringify(writtenConfig.channels?.discord?.token)}`"
|
||||
- assert:
|
||||
expr: "!JSON.stringify(writtenConfig.channels?.discord ?? {}).includes(config.discordToken)"
|
||||
message: Crestodian persisted the raw Discord token.
|
||||
- set: auditText
|
||||
value:
|
||||
expr: "await fs.readFile(path.join(stateDir, 'audit', 'crestodian.jsonl'), 'utf8')"
|
||||
- forEach:
|
||||
items:
|
||||
- crestodian.setup
|
||||
- config.setDefaultModel
|
||||
- agents.create
|
||||
- config.setRef
|
||||
- config.set
|
||||
item: operation
|
||||
actions:
|
||||
- assert:
|
||||
expr: 'auditText.includes(`"operation":"${operation}"`)'
|
||||
message:
|
||||
expr: "`missing audit entry for ${operation}: ${auditText}`"
|
||||
detailsExpr: "`stateDir=${stateDir}\\nconfigPath=${configPath}\\nagent=${JSON.stringify(agent)}\\nDiscord SecretRef=${JSON.stringify(writtenConfig.channels?.discord?.token)}`"
|
||||
```
|
||||
@@ -108,6 +108,36 @@ async function main() {
|
||||
"Crestodian agent creation did not apply",
|
||||
);
|
||||
|
||||
clearConfigCache();
|
||||
const discordPluginAllowRuntime = createRuntime();
|
||||
await runCrestodian(
|
||||
{
|
||||
message: 'config set plugins.allow ["discord"]',
|
||||
yes: true,
|
||||
interactive: false,
|
||||
},
|
||||
discordPluginAllowRuntime.runtime,
|
||||
);
|
||||
assert(
|
||||
discordPluginAllowRuntime.lines.join("\n").includes("[crestodian] done: config.set"),
|
||||
"Crestodian Discord plugin allowlist did not apply",
|
||||
);
|
||||
|
||||
clearConfigCache();
|
||||
const discordPluginEntryRuntime = createRuntime();
|
||||
await runCrestodian(
|
||||
{
|
||||
message: "config set plugins.entries.discord.enabled true",
|
||||
yes: true,
|
||||
interactive: false,
|
||||
},
|
||||
discordPluginEntryRuntime.runtime,
|
||||
);
|
||||
assert(
|
||||
discordPluginEntryRuntime.lines.join("\n").includes("[crestodian] done: config.set"),
|
||||
"Crestodian Discord plugin entry did not apply",
|
||||
);
|
||||
|
||||
clearConfigCache();
|
||||
const discordTokenRuntime = createRuntime();
|
||||
await runCrestodian(
|
||||
@@ -162,6 +192,11 @@ async function main() {
|
||||
assert(reef, "Crestodian did not create reef agent");
|
||||
assert(reef.workspace === "/tmp/openclaw-reef", "Crestodian did not write reef workspace");
|
||||
assert(reef.model === "openai/gpt-5.2", "Crestodian did not write reef model");
|
||||
assert(config.plugins?.allow?.includes("discord"), "Crestodian did not allow Discord plugin");
|
||||
assert(
|
||||
config.plugins?.entries?.discord?.enabled === true,
|
||||
"Crestodian did not enable Discord plugin entry",
|
||||
);
|
||||
assert(config.channels?.discord?.enabled === true, "Crestodian did not enable Discord");
|
||||
const discordToken = config.channels?.discord?.token;
|
||||
assert(
|
||||
|
||||
Reference in New Issue
Block a user