vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 14:10:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(plugins): split kitchen sink qa personalities (#74612)

Browse Source
This commit is contained in:
Vincent Koc
2026-04-29 13:56:49 -07:00
committed by GitHub
parent 987af6805b
commit 1a4d55de43
6 changed files with 155 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/e2e/kitchen-sink-plugin-docker.sh
View File

@@ -10,7 +10,9 @@ OPENCLAW_TEST_STATE_SCRIPT_B64="$(docker_e2e_test_state_shell_b64 kitchen-sink-p
DEFAULT_KITCHEN_SINK_SCENARIOS="$(
cat <<'SCENARIOS'
npm-latest|npm:@openclaw/kitchen-sink@latest|openclaw-kitchen-sink-fixture|npm|success|full
npm-latest-full|npm:@openclaw/kitchen-sink@latest|openclaw-kitchen-sink-fixture|npm|success|full
npm-latest-conformance|npm:@openclaw/kitchen-sink@latest|openclaw-kitchen-sink-fixture|npm|success|conformance|conformance
npm-latest-adversarial|npm:@openclaw/kitchen-sink@latest|openclaw-kitchen-sink-fixture|npm|success|adversarial|adversarial
npm-beta|npm:@openclaw/kitchen-sink@beta|openclaw-kitchen-sink-fixture|npm|failure|none
clawhub-latest|clawhub:openclaw-kitchen-sink@latest|openclaw-kitchen-sink-fixture|clawhub|success|basic
clawhub-beta|clawhub:openclaw-kitchen-sink@beta|openclaw-kitchen-sink-fixture|clawhub|failure|none

85
scripts/e2e/lib/kitchen-sink-plugin/assertions.mjs
View File

@@ -87,11 +87,20 @@ function readConfig() {
function configureRuntime() {
const pluginId = process.env.KITCHEN_SINK_ID;
const personality = process.env.KITCHEN_SINK_PERSONALITY;
const { configPath, config } = readConfig();
config.plugins = config.plugins || {};
config.plugins.entries = config.plugins.entries || {};
config.plugins.entries[pluginId] = {
...config.plugins.entries[pluginId],
...(personality
? {
config: {
...config.plugins.entries[pluginId]?.config,
personality,
},
}
: {}),
hooks: {
...config.plugins.entries[pluginId]?.hooks,
allowConversationAccess: true,
@@ -124,6 +133,39 @@ const expectMissing = (listValue, expected, field) => {
}
};
function assertExpectedDiagnostics(surfaceMode, errorMessages) {
const expectedErrorMessages = new Set([
"only bundled plugins can register agent tool result middleware",
'agent harness "kitchen-sink-agent-harness" registration missing required runtime methods',
'channel "kitchen-sink-channel-probe" registration missing required config helpers',
"cli registration missing explicit commands metadata",
"only bundled plugins can register Codex app-server extension factories",
'compaction provider "kitchen-sink-compaction-provider" registration missing summarize',
"context engine registration missing id",
"http route registration missing or invalid auth: /kitchen-sink/http-route",
"plugin must own memory slot or declare contracts.memoryEmbeddingProviders for adapter: kitchen-sink-memory-embedding-provider",
"memory prompt supplement registration missing builder",
]);
if (surfaceMode !== "full" && surfaceMode !== "adversarial") {
if (errorMessages.size > 0) {
throw new Error(
`unexpected kitchen-sink diagnostic errors: ${[...errorMessages].join(", ")}`,
);
}
return;
}
for (const message of errorMessages) {
if (!expectedErrorMessages.has(message)) {
throw new Error(`unexpected kitchen-sink diagnostic error: ${message}`);
}
}
for (const message of expectedErrorMessages) {
if (!errorMessages.has(message)) {
throw new Error(`missing expected kitchen-sink diagnostic error: ${message}`);
}
}
}
function assertRealPathInside(parentPath, childPath, label) {
const parentRealPath = fs.realpathSync(parentPath);
const childRealPath = fs.realpathSync(childPath);
@@ -181,8 +223,10 @@ function assertInstalled() {
);
}
expectIncludes(inspect.plugin?.channelIds, "kitchen-sink-channel", "channels");
expectIncludes(inspect.plugin?.providerIds, "kitchen-sink-provider", "providers");
if (surfaceMode !== "adversarial") {
expectIncludes(inspect.plugin?.channelIds, "kitchen-sink-channel", "channels");
expectIncludes(inspect.plugin?.providerIds, "kitchen-sink-provider", "providers");
}
const diagnostics = [
...(list.diagnostics || []),
@@ -193,7 +237,7 @@ function assertInstalled() {
diagnostics.filter((diag) => diag?.level === "error").map((diag) => String(diag.message || "")),
);
if (surfaceMode === "full") {
if (surfaceMode === "full" || surfaceMode === "conformance") {
const toolNames = Array.isArray(inspect.tools)
? inspect.tools.flatMap((entry) => (Array.isArray(entry?.names) ? entry.names : []))
: [];
@@ -232,8 +276,13 @@ function assertInstalled() {
}
expectMissing(inspect.plugin?.agentHarnessIds, "kitchen-sink-agent-harness", "agent harnesses");
expectIncludes(inspect.services, "kitchen-sink-service", "services");
expectIncludes(inspect.commands, "kitchen-sink-command", "commands");
expectIncludes(toolNames, "kitchen-sink-tool", "tools");
if (surfaceMode === "full") {
expectIncludes(inspect.commands, "kitchen-sink-command", "commands");
expectIncludes(toolNames, "kitchen-sink-tool", "tools");
} else {
expectIncludes(inspect.commands, "kitchen", "commands");
expectIncludes(toolNames, "kitchen_sink_text", "tools");
}
if (
(inspect.plugin?.hookCount || 0) < 30 ||
!Array.isArray(inspect.typedHooks) ||
@@ -243,32 +292,8 @@ function assertInstalled() {
`expected kitchen-sink typed hooks to load, got hookCount=${inspect.plugin?.hookCount} typedHooks=${inspect.typedHooks?.length}`,
);
}
const expectedErrorMessages = new Set([
"only bundled plugins can register agent tool result middleware",
'agent harness "kitchen-sink-agent-harness" registration missing required runtime methods',
'channel "kitchen-sink-channel-probe" registration missing required config helpers',
"cli registration missing explicit commands metadata",
"only bundled plugins can register Codex app-server extension factories",
'compaction provider "kitchen-sink-compaction-provider" registration missing summarize',
"context engine registration missing id",
"http route registration missing or invalid auth: /kitchen-sink/http-route",
"plugin must own memory slot or declare contracts.memoryEmbeddingProviders for adapter: kitchen-sink-memory-embedding-provider",
"memory prompt supplement registration missing builder",
]);
for (const message of errorMessages) {
if (!expectedErrorMessages.has(message)) {
throw new Error(`unexpected kitchen-sink diagnostic error: ${message}`);
}
}
for (const message of expectedErrorMessages) {
if (!errorMessages.has(message)) {
throw new Error(`missing expected kitchen-sink diagnostic error: ${message}`);
}
}
} else if (errorMessages.size > 0) {
throw new Error(`unexpected kitchen-sink diagnostic errors: ${[...errorMessages].join(", ")}`);
}
assertExpectedDiagnostics(surfaceMode, errorMessages);
const indexPath = path.join(process.env.HOME, ".openclaw", "plugins", "installs.json");
const index = readJson(indexPath);

3
scripts/e2e/lib/kitchen-sink-plugin/sweep.sh
View File

@@ -108,7 +108,7 @@ if [[ "$KITCHEN_SINK_SCENARIOS" == *"clawhub:"* ]] &&
fi
scenario_count=0
while IFS='|' read -r label spec plugin_id source expectation surface_mode; do
while IFS='|' read -r label spec plugin_id source expectation surface_mode personality; do
if [ -z "${label:-}" ] || [[ "$label" == \#* ]]; then
continue
fi
@@ -118,6 +118,7 @@ while IFS='|' read -r label spec plugin_id source expectation surface_mode; do
export KITCHEN_SINK_ID="$plugin_id"
export KITCHEN_SINK_SOURCE="$source"
export KITCHEN_SINK_SURFACE_MODE="$surface_mode"
export KITCHEN_SINK_PERSONALITY="${personality:-}"
case "$expectation" in
success)
run_success_scenario