diff --git a/.agents/skills/autoreview/scripts/autoreview b/.agents/skills/autoreview/scripts/autoreview index 2e2b50d49752..79169878bc00 100755 --- a/.agents/skills/autoreview/scripts/autoreview +++ b/.agents/skills/autoreview/scripts/autoreview @@ -278,11 +278,16 @@ class ReviewChunk(NamedTuple): context: str = "" SECRET_PLACEHOLDER_VALUES = { "changeme", + "decoy-token", "dummy", "example", "fake", "gateway-token", + "matrix_qa_e2ee_cli_gateway", + "matrix_qa_e2ee_thread", + "matrix_qa_e2ee_verify_notice", "not-a-real", + "not-a-valid-matrix-recovery-key", "placeholder", "redacted", "sample", @@ -455,6 +460,27 @@ BACKTICK_TEMPLATE_INTERPOLATION_PATTERN = re.compile(r"\$\{([^{}\r\n]+)\}") BACKTICK_TEMPLATE_SAFE_LITERAL_PATTERN = re.compile( r"(?i)(?:(?:Bearer|Basic)[ \t]+|[ \t:./,_-]*)" ) +BACKTICK_TEMPLATE_FIXTURE_PREFIX_VALUES = {"matrix-qa-"} +SOURCE_CODE_REFERENCE_VALUES = { + "cliDevice.accessToken", + "context.driverAccessToken", + "context.driverPassword", + "context.observerAccessToken", + "context.observerPassword", + "context.sutAccessToken", + "context.sutPassword", + "driverAccount.accessToken", + "driverAccount.password", + "driverPassword", + "recoveryDevice.accessToken", + "recoveryDevice.password", +} +SOURCE_CODE_REFERENCE_PATTERN = re.compile( + r"(? bool: return False -def fallback_expression(text: str) -> str: - operator = re.match(r"\s*(?:\|\||&&|\?\?|\+|\?|or\b|and\b|if\b|unless\b)", text) +def fallback_expression(text: str, *, typescript: bool = False) -> str: + operator_keywords = ( + r"|as(?![\w$])|satisfies(?![\w$])" if typescript else "" + ) + operator = re.match( + r"\s*(?:\|\||&&|\?\?|[=<>!~:+\-*/%&|^?.,]" + r"|and\b|else\b|if\b|in(?![\w$])|instanceof(?![\w$])" + r"|is\b|not\b|or\b" + r"|unless\b" + + operator_keywords + + r")", + text, + ) cursor = operator.end() if operator is not None else 0 stack: list[str] = [] operand_started = False @@ -1920,9 +1957,16 @@ def fallback_expression(text: str) -> str: char = text[cursor] next_char = text[cursor + 1] if cursor + 1 < len(text) else "" if line_comment: - if char == "\n": + if char in "\r\n\u2028\u2029": line_comment = False - if operand_started and not stack: + if ( + operand_started + and not stack + and not javascript_continues_after_line_terminator( + text[cursor + 1 :], + typescript=typescript, + ) + ): break cursor += 1 continue @@ -1974,14 +2018,54 @@ def fallback_expression(text: str) -> str: continue if not stack and char in ",;)]}": break - if char == "\n" and operand_started and not stack: - break - if not char.isspace(): + if char in "\r\n\u2028\u2029" and operand_started and not stack: + if not javascript_continues_after_line_terminator( + text[cursor + 1 :], + typescript=typescript, + ): + break + if not stack and (char.isalpha() or char in "_$"): + word_end = cursor + 1 + while word_end < len(text) and ( + text[word_end].isalnum() or text[word_end] in "_$" + ): + word_end += 1 + word = text[cursor:word_end] + word_operators = {"in", "instanceof"} + if typescript: + word_operators.update({"as", "satisfies"}) + operand_started = word not in word_operators + cursor = word_end + continue + if not stack and char in "=<>!~:+-*/%&|^?.": + operand_started = False + elif not char.isspace(): operand_started = True cursor += 1 return text[:cursor] +def javascript_continues_after_line_terminator( + text: str, + *, + typescript: bool = False, +) -> bool: + suffix = text.lstrip() + if suffix.startswith(("++", "--", "~")) or ( + suffix.startswith("!") and not suffix.startswith("!=") + ): + return False + type_operators = ( + r"|as(?![\w$])|satisfies(?![\w$])" if typescript else "" + ) + return re.match( + r"(?:[=<>!:+\-*/%&|^?.,([`]|in(?![\w$])|instanceof(?![\w$])" + + type_operators + + r")", + suffix, + ) is not None + + def top_level_fallback_suffix( text: str, *, @@ -3717,9 +3801,14 @@ def secret_literal_risk(expression: str, minimum_length: int = 12) -> bool: return False -def fallback_secret_risk(text: str, minimum_length: int = 8) -> bool: +def fallback_secret_risk( + text: str, + minimum_length: int = 8, + *, + typescript: bool = False, +) -> bool: return secret_literal_risk( - fallback_expression(text), + fallback_expression(text, typescript=typescript), minimum_length=minimum_length, ) @@ -4610,27 +4699,37 @@ def safe_secret_call_suffix(text: str, end: int, call_target: str) -> bool: return safe_secret_assignment_suffix(text, cursor) +def safe_backtick_literal_segment(value: str) -> bool: + return bool( + BACKTICK_TEMPLATE_SAFE_LITERAL_PATTERN.fullmatch(value) + or value.casefold() in BACKTICK_TEMPLATE_FIXTURE_PREFIX_VALUES + ) + + +def safe_backtick_interpolation_expression(expression: str) -> bool: + quoted_reference = "${" + expression + "}" + if any( + pattern.fullmatch(quoted_reference) + for pattern in QUOTED_SECRET_REFERENCE_PATTERNS + ): + return True + return re.fullmatch(r"(?:crypto\.)?randomUUID\(\)", expression) is not None + + def safe_backtick_secret_template(value: str) -> bool: cursor = 0 found_interpolation = False for match in BACKTICK_TEMPLATE_INTERPOLATION_PATTERN.finditer(value): literal = value[cursor : match.start()] - if BACKTICK_TEMPLATE_SAFE_LITERAL_PATTERN.fullmatch(literal) is None: + if not safe_backtick_literal_segment(literal): return False expression = match.group(1).strip() - quoted_reference = "${" + expression + "}" - if not any( - pattern.fullmatch(quoted_reference) - for pattern in QUOTED_SECRET_REFERENCE_PATTERNS - ): + if not safe_backtick_interpolation_expression(expression): return False found_interpolation = True cursor = match.end() literal = value[cursor:] - return ( - found_interpolation - and BACKTICK_TEMPLATE_SAFE_LITERAL_PATTERN.fullmatch(literal) is not None - ) + return found_interpolation and safe_backtick_literal_segment(literal) def javascript_template_literal_end( @@ -4764,6 +4863,82 @@ def mask_reference_declaration_evidence(text: str) -> str: return mask_csharp_evidence_prefix("".join(masked)) +@functools.lru_cache(maxsize=8) +def javascript_reference_spans(text: str) -> frozenset[tuple[int, int]]: + return frozenset( + (match.start(), match.end()) + for match in SOURCE_CODE_REFERENCE_PATTERN.finditer(text) + ) + + +def javascript_source_whitespace(char: str) -> bool: + return ( + char in "\t\v\f \r\n\u2028\u2029\ufeff" + or unicodedata.category(char) == "Zs" + ) + + +def safe_javascript_reference_suffix( + text: str, + end: int, + *, + typescript: bool, +) -> bool: + line_terminators = "\r\n\u2028\u2029" + cursor = end + saw_newline = False + while cursor < len(text): + while cursor < len(text) and javascript_source_whitespace(text[cursor]): + saw_newline = saw_newline or text[cursor] in line_terminators + cursor += 1 + if text.startswith("//", cursor): + newline = re.search(r"[\r\n\u2028\u2029]", text[cursor + 2 :]) + if newline is None: + return True + cursor += 2 + newline.start() + saw_newline = True + continue + if text.startswith("/*", cursor): + comment_end = text.find("*/", cursor + 2) + if comment_end < 0: + return True + comment = text[cursor : comment_end + 2] + saw_newline = saw_newline or any( + terminator in comment for terminator in line_terminators + ) + cursor = comment_end + 2 + continue + break + if cursor >= len(text): + return True + if text[cursor] in ";}])" or text[cursor] == ",": + return True + if saw_newline and ( + text.startswith(("++", "--"), cursor) + or text[cursor] == "~" + or (text[cursor] == "!" and not text.startswith("!=", cursor)) + ): + return True + continuation_keyword = re.match( + ( + r"(?:as|in|instanceof|satisfies)(?![\w$])" + if typescript + else r"(?:in|instanceof)(?![\w$])" + ), + text[cursor:], + ) + continuation = ( + text[cursor] in "=<>!~:+-*/%&|^?.,([`" + or continuation_keyword is not None + ) + if continuation: + return not fallback_secret_risk( + text[cursor:], + typescript=typescript, + ) + return saw_newline + + def bare_code_reference( text: str, start: int, @@ -4848,7 +5023,11 @@ def basic_authorization_risk(text: str) -> bool: return False -def secret_text_risk(text: str) -> bool: +def secret_text_risk( + text: str, + *, + javascript_dialect: str | None = None, +) -> bool: uri_authorities = uri_authority_ranges(text) if credentialed_uri_risk(text, uri_authorities) or basic_authorization_risk(text) or any( pattern.search(text) for pattern in SECRET_VALUE_PATTERNS @@ -4869,6 +5048,11 @@ def secret_text_risk(text: str) -> bool: text, {prefix.start() for prefix in assignment_prefixes}, ) + source_reference_spans = ( + javascript_reference_spans(text) + if javascript_dialect is not None + else frozenset() + ) for prefix in assignment_prefixes: fallback = top_level_fallback_suffix( text[prefix.end() :], @@ -4945,6 +5129,18 @@ def secret_text_risk(text: str) -> bool: and safe_secret_assignment_suffix(text, match.end()) ): continue + if not quoted: + source_start = match.end() - len(value) + source_end = match.end() - (1 if value.endswith("!") else 0) + if ( + (source_start, source_end) in source_reference_spans + and safe_javascript_reference_suffix( + text, + match.end(), + typescript=javascript_dialect == "typescript", + ) + ): + continue reference_patterns = ( QUOTED_SECRET_REFERENCE_PATTERNS if quoted @@ -4991,8 +5187,13 @@ def secret_text_risk(text: str) -> bool: return False -def require_no_secret_values(label: str, text: str) -> None: - if secret_text_risk(text): +def require_no_secret_values( + label: str, + text: str, + *, + javascript_dialect: str | None = None, +) -> None: + if secret_text_risk(text, javascript_dialect=javascript_dialect): raise SystemExit( "refusing to include secret-like content in review bundle; " f"clean or redact {label} before running autoreview" @@ -5177,6 +5378,82 @@ def tracked_sensitive_repo_path_risk(rel: str) -> str | None: return None +def javascript_review_dialect(rel: str) -> str | None: + suffix = Path(rel).suffix.lower() + if suffix in {".cts", ".mts", ".ts", ".tsx"}: + return "typescript" + if suffix in {".cjs", ".js", ".jsx", ".mjs"}: + return "javascript" + return None + + +def git_c_unquote(value: str) -> str | None: + if len(value) < 2 or value[0] != '"' or value[-1] != '"': + return None + escapes = { + "a": 7, + "b": 8, + "f": 12, + "n": 10, + "r": 13, + "t": 9, + "v": 11, + "\\": 92, + '"': 34, + } + decoded = bytearray() + cursor = 1 + while cursor < len(value) - 1: + char = value[cursor] + if char != "\\": + decoded.extend(char.encode("utf-8")) + cursor += 1 + continue + cursor += 1 + if cursor >= len(value) - 1: + return None + escape = value[cursor] + if escape in escapes: + decoded.append(escapes[escape]) + cursor += 1 + continue + octal = re.match(r"[0-7]{1,3}", value[cursor:-1]) + if octal is None: + return None + decoded.append(int(octal.group(0), 8)) + cursor += octal.end() + try: + return decoded.decode("utf-8") + except UnicodeDecodeError: + return None + + +def diff_marker_path(value: str) -> str | None: + if value == "/dev/null": + return None + if value.startswith('"'): + decoded = git_c_unquote(value) + if decoded is None: + return None + value = decoded + if not value.startswith(("a/", "b/")): + return None + return value[2:] + + +def diff_section_paths(section: str) -> tuple[str | None, str | None]: + old_path: str | None = None + new_path: str | None = None + for line in section.splitlines(): + if line.startswith("@@"): + break + if line.startswith("--- "): + old_path = diff_marker_path(line[4:]) + elif line.startswith("+++ "): + new_path = diff_marker_path(line[4:]) + return old_path, new_path + + def validate_review_patch( label: str, paths: list[str], @@ -5202,8 +5479,33 @@ def validate_review_patch( f"({patch_bytes} bytes; limit {limit}); split the change into smaller review targets" ) require_no_secret_values(label, unified_diff_metadata(patch)) - for content in unified_diff_contents(patch): - require_no_secret_values(label, content) + sections = [ + unit + for unit in review_bundle_units(patch) + if unit.startswith("diff --git ") + ] + if sections: + expected_paths = set(paths) + for section in sections: + old_path, new_path = diff_section_paths(section) + old_content, new_content = unified_diff_contents(section) + for rel, content in ( + (old_path, old_content), + (new_path, new_content), + ): + javascript_dialect = ( + javascript_review_dialect(rel) + if rel is not None and rel in expected_paths + else None + ) + require_no_secret_values( + f"{label} {rel or ''}", + content, + javascript_dialect=javascript_dialect, + ) + else: + for content in unified_diff_contents(patch): + require_no_secret_values(label, content) return patch @@ -5312,7 +5614,10 @@ def file_bundle_snapshot( text = data.decode("utf-8") except UnicodeDecodeError: return "", True, "non-UTF-8 file" - if secret_text_risk(text): + if secret_text_risk( + text, + javascript_dialect=javascript_review_dialect(rel), + ): return "", True, "secret-like content" return text, False, None diff --git a/.agents/skills/autoreview/tests/test_autoreview_hardening.py b/.agents/skills/autoreview/tests/test_autoreview_hardening.py index fb332ee9243b..8973584b35cf 100644 --- a/.agents/skills/autoreview/tests/test_autoreview_hardening.py +++ b/.agents/skills/autoreview/tests/test_autoreview_hardening.py @@ -1859,6 +1859,562 @@ class AutoreviewHardeningTests(unittest.TestCase): ) ) + def test_secret_detector_allows_typescript_object_secret_references(self) -> None: + content = ( + "async function configure(context: RuntimeContext) {\n" + " const cliDevice = await login();\n" + " const driverPassword = readPassword();\n" + " return {\n" + " access" + + "Token" + + ": cliDevice.access" + + "Token,\n" + + " pass" + + "word: driverPass" + + "word,\n" + + " ...(context.driverPass" + + "word ? { pass" + + "word: context.driverPass" + + "word } : {}),\n" + + " };\n" + + "}" + ) + yaml_literal = "pass" + "word: actualProductionSecret," + yaml_reference = "pass" + "word: context.driverPass" + "word" + yaml_flow_reference = ( + "{ pass" + "word: context.driverPass" + "word, enabled: true }" + ) + sut_reference = ( + "const pass" + + "word = context.sutPass" + + "word;" + ) + literal_value = "actual-production-" + "secret" + source_literal = ( + "function configure() { return { pass" + + 'word: "' + + literal_value + + '" }; }' + ) + jwt_like = "eyJhbGciOiJIUzI1NiJ9" + ".payload." + "signature" + undeclared_member = ( + "const config = { pass" + + "word: " + + jwt_like + + " };" + ) + jwt_root = jwt_like.split(".", 1)[0] + declared_member = ( + f"const {jwt_root} = {{}};\n" + + "const config = { pass" + + "word: " + + jwt_like + + " };" + ) + prefixed_member = ( + "const session = {};\n" + + "const config = { pass" + + "word: session." + + jwt_like + + " };" + ) + declared_identifier = "CorrectHorseBattery" + "Staple123" + declared_identifier_value = ( + f"const {declared_identifier} = 0;\n" + + "const config = { pass" + + "word: " + + declared_identifier + + " };" + ) + suffixed_reference = ( + "const config = { pass" + + "word: context.driverPass" + + "wordExtra };" + ) + prefixed_reference = ( + "const config = { pass" + + "word: xcontext.driverPass" + + "word };" + ) + final_property = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + "word }; }" + ) + inline_property = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + "word, enabled: true }; }" + ) + asserted_property = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + "word! }; }" + ) + cast_property = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + "word as string }; }" + ) + union_cast_property = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + "word as string | undefined }; }" + ) + next_statement = ( + "function configure(context: RuntimeContext) {\n" + + " const pass" + + "word = context.driverPass" + + "word\n" + + " return consume();\n" + + "}" + ) + line_comment = ( + "const pass" + + "word = context.driverPass" + + "word // supplied by CI\n" + + "consume();" + ) + block_comment = ( + "const pass" + + "word = context.driverPass" + + "word /* supplied by CI */;" + ) + concatenated_literal = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + 'word + "' + + literal_value + + '" }; }' + ) + continued_literal = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + "word\n" + + ' + "' + + literal_value + + '" }; }' + ) + fallback_literal = ( + "function configure(context: RuntimeContext) { return { pass" + + "word: context.driverPass" + + 'word ?? "' + + literal_value + + '" }; }' + ) + assigned_literal = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + ' = "' + + literal_value + + '";' + ) + newline_cast_literal = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + " as string + \"" + + literal_value + + '";' + ) + commented_continuation = ( + "const pass" + + "word = context.driverPass" + + "word // supplied by CI\n" + + ' + "' + + literal_value + + '";' + ) + unicode_comment_continuation = ( + "const pass" + + "word = context.driverPass" + + "word // supplied by CI" + + chr(0x2028) + + ' + "' + + literal_value + + '";' + ) + unicode_space_continuation = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + chr(0x00A0) + + '+ "' + + literal_value + + '";' + ) + multiline_cast_literal = ( + "const pass" + + "word = context.driverPass" + + "word as string\n" + + '+ "' + + literal_value + + '";' + ) + leading_comma_statement = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + ', username = "' + + literal_value + + '";' + ) + unary_statement = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + '!audit("' + + literal_value + + '");' + ) + inequality_literal = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + '!== "' + + literal_value + + '";' + ) + member_call_literal = ( + "const pass" + + "word = context.driverPass" + + 'word["concat"]("safe", "' + + literal_value + + '");' + ) + continued_then_unary_statement = ( + "const pass" + + "word = context.driverPass" + + "word + suffix\n" + + '!audit("' + + literal_value + + '");' + ) + trailing_operator_literal = ( + "const pass" + + "word = context.driverPass" + + "word + suffix +\n" + + ' "' + + literal_value + + '";' + ) + plain_javascript_as_statement = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + 'as("' + + literal_value + + '");' + ) + typescript_dollar_identifier = ( + "const pass" + + "word = context.driverPass" + + "word\n" + + 'as$logger("' + + literal_value + + '");' + ) + + self.assertFalse( + self.helper["secret_text_risk"]( + content, + javascript_dialect="typescript", + ) + ) + self.assertFalse( + self.helper["secret_text_risk"]( + sut_reference, + javascript_dialect="typescript", + ) + ) + self.assertTrue(self.helper["secret_text_risk"](sut_reference)) + self.assertFalse( + self.helper["secret_text_risk"]( + plain_javascript_as_statement, + javascript_dialect="javascript", + ) + ) + self.assertFalse( + self.helper["secret_text_risk"]( + typescript_dollar_identifier, + javascript_dialect="typescript", + ) + ) + self.assertFalse( + self.helper["secret_text_risk"]( + final_property, + javascript_dialect="typescript", + ) + ) + for source_reference in ( + inline_property, + asserted_property, + cast_property, + union_cast_property, + next_statement, + line_comment, + block_comment, + leading_comma_statement, + unary_statement, + continued_then_unary_statement, + ): + with self.subTest(source_reference=source_reference): + self.assertFalse( + self.helper["secret_text_risk"]( + source_reference, + javascript_dialect="typescript", + ) + ) + for unsafe_source_reference in ( + concatenated_literal, + continued_literal, + fallback_literal, + assigned_literal, + newline_cast_literal, + commented_continuation, + unicode_comment_continuation, + unicode_space_continuation, + multiline_cast_literal, + inequality_literal, + member_call_literal, + trailing_operator_literal, + ): + with self.subTest(unsafe_source_reference=unsafe_source_reference): + self.assertTrue( + self.helper["secret_text_risk"]( + unsafe_source_reference, + javascript_dialect="typescript", + ) + ) + self.assertTrue(self.helper["secret_text_risk"](yaml_literal)) + self.assertTrue(self.helper["secret_text_risk"](yaml_reference)) + self.assertTrue(self.helper["secret_text_risk"](yaml_flow_reference)) + self.assertTrue(self.helper["secret_text_risk"](source_literal)) + self.assertTrue(self.helper["secret_text_risk"](undeclared_member)) + self.assertTrue(self.helper["secret_text_risk"](declared_member)) + self.assertTrue(self.helper["secret_text_risk"](prefixed_member)) + self.assertTrue( + self.helper["secret_text_risk"](declared_identifier_value) + ) + self.assertTrue( + self.helper["secret_text_risk"]( + suffixed_reference, + javascript_dialect="typescript", + ) + ) + self.assertTrue( + self.helper["secret_text_risk"]( + prefixed_reference, + javascript_dialect="typescript", + ) + ) + + def test_review_patch_scopes_source_references_to_typescript_files(self) -> None: + property_name = "pass" + "word" + reference = "context.driverPass" + "word" + source_patch = ( + "diff --git a/src/runtime.ts b/src/runtime.ts\n" + "--- a/src/runtime.ts\n" + "+++ b/src/runtime.ts\n" + "@@ -0,0 +1 @@\n" + "+function configure(context: RuntimeContext) { return { " + + property_name + + ": " + + reference + + " }; }\n" + ) + narrow_source_patch = ( + "diff --git a/src/runtime.ts b/src/runtime.ts\n" + "--- a/src/runtime.ts\n" + "+++ b/src/runtime.ts\n" + "@@ -40,2 +40,3 @@ function configure(context: RuntimeContext) {\n" + " return {\n" + "+ " + + property_name + + ": " + + reference + + ",\n" + " };\n" + ) + config_patch = ( + "diff --git a/config.yml b/config.yml\n" + "--- a/config.yml\n" + "+++ b/config.yml\n" + "@@ -0,0 +1 @@\n" + "+" + + property_name + + ": " + + reference + + "\n" + ) + + self.assertEqual( + self.helper["validate_review_patch"]( + "local staged diff", + ["src/runtime.ts"], + source_patch, + ), + source_patch, + ) + self.assertEqual( + self.helper["validate_review_patch"]( + "local staged diff", + ["src/runtime.ts"], + narrow_source_patch, + ), + narrow_source_patch, + ) + with self.assertRaisesRegex(SystemExit, "secret-like content"): + self.helper["validate_review_patch"]( + "local staged diff", + ["src/runtime.ts", "config.yml"], + source_patch + config_patch, + ) + with self.assertRaisesRegex(SystemExit, "secret-like content"): + self.helper["validate_review_patch"]( + "local staged diff", + ["config.yml", "src/runtime.ts"], + source_patch + config_patch, + ) + + def test_review_patch_scans_rename_sides_with_their_own_file_types(self) -> None: + property_name = "pass" + "word" + reference = "context.driverPass" + "word" + patch = ( + "diff --git a/src/runtime.ts b/config.yml\n" + "similarity index 80%\n" + "rename from src/runtime.ts\n" + "rename to config.yml\n" + "--- a/src/runtime.ts\n" + "+++ b/config.yml\n" + "@@ -1 +1 @@\n" + "-function configure(context: RuntimeContext) { return { " + + property_name + + ": " + + reference + + " }; }\n" + "+" + + property_name + + ": " + + reference + + "\n" + ) + + with self.assertRaisesRegex(SystemExit, "secret-like content"): + self.helper["validate_review_patch"]( + "branch diff", + ["src/runtime.ts", "config.yml"], + patch, + ) + + def test_review_patch_decodes_git_quoted_source_paths(self) -> None: + property_name = "pass" + "word" + reference = "context.driverPass" + "word" + patch = ( + 'diff --git "a/\\303\\251.ts" "b/\\303\\251.ts"\n' + '--- "a/\\303\\251.ts"\n' + '+++ "b/\\303\\251.ts"\n' + "@@ -40,2 +40,3 @@ function configure(context: RuntimeContext) {\n" + " return {\n" + "+ " + + property_name + + ": " + + reference + + ",\n" + " };\n" + ) + + self.assertEqual( + self.helper["validate_review_patch"]( + "local staged diff", + ["é.ts"], + patch, + ), + patch, + ) + self.assertEqual( + self.helper["javascript_review_dialect"]("module.mts"), + "typescript", + ) + self.assertEqual( + self.helper["javascript_review_dialect"]("module.cts"), + "typescript", + ) + + def test_secret_detector_allows_generated_fixture_credentials(self) -> None: + property_name = "pass" + "word" + variable_name = "to" + "ken" + access_property = "access" + "Token" + generated_fixture = ( + f"function register() {{ return {{ {property_name}: " + + "`matrix-qa-${randomUUID()}` }; }" + ) + generated_marker = ( + f"const {variable_name} = " + + 'buildMatrixQaToken("MATRIX_QA_E2EE_THREAD");' + ) + decoy_fixture = ( + f"const config = {{ {access_property}: " + + 'decoy-' + + 'token" };' + ) + invalid_recovery_fixture = ( + 'const recoveryKey = "not-' + + 'a-valid-matrix-recovery-key";' + ) + literal_value = "actual-production-" + "secret" + fixture_shaped_literal = "PROD_TEST_ACTUAL_" + "SECRET_0123456789" + adversarial_label = "TEST_Q7WX9M2NK4PV8R6DH3JC" + unsafe_template = ( + f"const {property_name} = " + + "`prod-live-secret-${randomUUID()}`;" + ) + unsafe_string_template = ( + f"const {property_name} = " + + "`prod-test-live-secret-${String()}`;" + ) + unsafe_suffix_template = ( + f"const {property_name} = " + + "`prod-live-secret-test-${randomUUID()}`;" + ) + unsafe_call = ( + f"const {variable_name} = " + + f'buildToken("{literal_value}");' + ) + unsafe_fixture_label = ( + f"const {property_name} = " + + f'"{fixture_shaped_literal}";' + ) + unsafe_generator_label = ( + f"const {variable_name} = " + + f'buildMatrixQaToken("{fixture_shaped_literal}");' + ) + unsafe_identity_call = ( + f"const {variable_name} = " + + f'buildTestToken("{adversarial_label}");' + ) + + for content in ( + generated_fixture, + generated_marker, + decoy_fixture, + invalid_recovery_fixture, + ): + with self.subTest(content=content): + self.assertFalse(self.helper["secret_text_risk"](content)) + for content in ( + unsafe_template, + unsafe_string_template, + unsafe_suffix_template, + unsafe_call, + unsafe_fixture_label, + unsafe_generator_label, + unsafe_identity_call, + ): + with self.subTest(content=content): + self.assertTrue(self.helper["secret_text_risk"](content)) + def test_fallback_self_test_ignores_ambient_model_overrides(self) -> None: with mock.patch.dict( os.environ, diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 8d2e2d1ef7a2..def23b2e36e5 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -273,7 +273,6 @@ extensions/qa-matrix/src/runners/contract/runtime.ts extensions/qa-matrix/src/runners/contract/scenario-catalog.ts extensions/qa-matrix/src/runners/contract/scenario-runtime-approval.ts extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-destructive.ts -extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts extensions/qa-matrix/src/runners/contract/scenarios.test.ts extensions/qa-matrix/src/substrate/client.ts diff --git a/docs/channels/matrix.md b/docs/channels/matrix.md index 0e8ef5e8d057..ff3aa60f20a5 100644 --- a/docs/channels/matrix.md +++ b/docs/channels/matrix.md @@ -299,11 +299,12 @@ All `openclaw matrix` commands accept `--verbose` (full diagnostics), `--json` ( ```bash openclaw matrix encryption setup +printf '%s\n' "$MATRIX_RECOVERY_KEY" | openclaw matrix encryption setup --recovery-key-stdin ``` Bootstraps secret storage and cross-signing, creates a room-key backup if needed, then prints status and next steps. Useful flags: -- `--recovery-key ` apply a recovery key before bootstrapping (prefer the stdin form below) +- `--recovery-key-stdin` reads a recovery key from stdin without exposing it in process arguments; `--recovery-key ` remains available for compatibility - `--force-reset-cross-signing` discard the current cross-signing identity and create a new one (intentional use only) For a new account, enable E2EE at creation time: diff --git a/extensions/matrix/src/cli.test.ts b/extensions/matrix/src/cli.test.ts index 3e55f2d3e730..55af7762ab17 100644 --- a/extensions/matrix/src/cli.test.ts +++ b/extensions/matrix/src/cli.test.ts @@ -1183,7 +1183,7 @@ describe("matrix CLI verification commands", () => { expect(console.log).toHaveBeenCalledWith("Matrix verification bootstrap: complete"); }); - it("enables E2EE and prints verification status from matrix encryption setup", async () => { + it("reads the recovery key from stdin during matrix encryption setup", async () => { const cfg = { channels: { matrix: { @@ -1218,11 +1218,13 @@ describe("matrix CLI verification commands", () => { mockMatrixVerificationStatus({ recoveryKeyCreatedAt: "2026-03-09T06:00:00.000Z", }); + mockRecoveryKeyStdin("stdin-recovery-key\n"); const program = buildProgram(); - await program.parseAsync(["matrix", "encryption", "setup", "--account", "ops"], { - from: "user", - }); + await program.parseAsync( + ["matrix", "encryption", "setup", "--account", "ops", "--recovery-key-stdin"], + { from: "user" }, + ); const replaceArg = mockCallArg(matrixRuntimeReplaceConfigFileMock) as { nextConfig?: CoreConfig; @@ -1239,7 +1241,7 @@ describe("matrix CLI verification commands", () => { }; expect(bootstrapArg.accountId).toBe("ops"); expect(bootstrapArg.cfg?.channels?.matrix?.accounts?.ops?.encryption).toBe(true); - expect(bootstrapArg.recoveryKey).toBeUndefined(); + expect(bootstrapArg.recoveryKey).toBe("stdin-recovery-key"); expect(bootstrapArg.forceResetCrossSigning).toBe(false); const statusArg = mockCallArg(getMatrixVerificationStatusMock) as Record; expect(statusArg.accountId).toBe("ops"); diff --git a/extensions/matrix/src/cli.ts b/extensions/matrix/src/cli.ts index 037b4ed282af..8775e5401d19 100644 --- a/extensions/matrix/src/cli.ts +++ b/extensions/matrix/src/cli.ts @@ -1634,7 +1634,11 @@ export function registerMatrixCli(params: { program: Command }): void { .command("setup") .description("Enable Matrix E2EE, bootstrap verification, and print next steps") .option("--account ", "Account ID (for multi-account setups)") - .option("--recovery-key ", "Recovery key to apply before bootstrap") + .option( + "--recovery-key ", + "Recovery key to apply before bootstrap (prefer --recovery-key-stdin)", + ) + .option("--recovery-key-stdin", "Read the Matrix recovery key from stdin") .option( "--force-reset-cross-signing", "Force reset cross-signing identity before bootstrap (requires active recovery key)", @@ -1645,6 +1649,7 @@ export function registerMatrixCli(params: { program: Command }): void { async (options: { account?: string; recoveryKey?: string; + recoveryKeyStdin?: boolean; forceResetCrossSigning?: boolean; verbose?: boolean; json?: boolean; @@ -1655,7 +1660,10 @@ export function registerMatrixCli(params: { program: Command }): void { run: async () => await setupMatrixEncryption({ account: options.account, - recoveryKey: options.recoveryKey, + recoveryKey: await resolveMatrixCliRecoveryKeyInput({ + recoveryKey: options.recoveryKey, + recoveryKeyStdin: options.recoveryKeyStdin, + }), forceResetCrossSigning: options.forceResetCrossSigning === true, }), onText: (result, verbose) => { diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-account.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-account.ts new file mode 100644 index 000000000000..bf977dd5719d --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-account.ts @@ -0,0 +1,417 @@ +// Qa Matrix plugin module implements account setup CLI E2EE scenarios. +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { startMatrixQaFaultProxy } from "../../substrate/fault-proxy.js"; +import { + assertMatrixQaCliAccountAddBootstrapStatus, + assertMatrixQaCliE2eeStatus, + buildMatrixQaCliE2eeAccountConfig, + runMatrixQaCliExpectedFailure, +} from "./scenario-runtime-e2ee-cli-config.js"; +import { createMatrixQaCliE2eeSetupRuntime } from "./scenario-runtime-e2ee-cli-runtime.js"; +import { + isMatrixQaCliBackupUsable, + parseMatrixQaCliJson, + registerMatrixQaCliE2eeAccount, + type MatrixQaCliAccountAddStatus, + type MatrixQaCliEncryptionSetupStatus, + type MatrixQaCliVerificationStatus, + writeMatrixQaCliOutputArtifacts, +} from "./scenario-runtime-e2ee-cli-shared.js"; +import { buildRoomKeyBackupUnavailableFaultRule } from "./scenario-runtime-e2ee-room.js"; +import { MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID } from "./scenario-runtime-e2ee-shared.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runMatrixQaE2eeCliAccountAddEnableE2eeScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli-add-e2ee"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Account Add Owner", + scenarioId: "matrix-e2ee-cli-account-add-enable-e2ee", + }); + const cli = await createMatrixQaCliE2eeSetupRuntime({ + artifactLabel: "cli-account-add-enable-e2ee", + context, + }); + try { + const addResult = await cli.run([ + "matrix", + "account", + "add", + "--account", + accountId, + "--name", + "Matrix QA CLI Account Add E2EE", + "--homeserver", + context.baseUrl, + "--user-id", + account.userId, + "--password", + account.password, + "--device-name", + "OpenClaw Matrix QA CLI Account Add E2EE", + "--allow-private-network", + "--enable-e2ee", + "--json", + ]); + const addArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "account-add-enable-e2ee", + result: addResult, + rootDir: cli.rootDir, + }); + const added = parseMatrixQaCliJson(addResult) as MatrixQaCliAccountAddStatus; + if (added.accountId !== accountId || added.encryptionEnabled !== true) { + throw new Error( + "Matrix CLI account add did not report E2EE enabled for the expected account", + ); + } + if (added.verificationBootstrap?.attempted !== true) { + throw new Error("Matrix CLI account add did not attempt verification bootstrap"); + } + if (added.verificationBootstrap.success !== true) { + throw new Error( + `Matrix CLI account add verification bootstrap failed: ${added.verificationBootstrap.error ?? "unknown error"}`, + ); + } + + const statusResult = await cli.run([ + "matrix", + "verify", + "status", + "--account", + accountId, + "--allow-degraded-local-state", + "--json", + ]); + const statusArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "verify-status", + result: statusResult, + rootDir: cli.rootDir, + }); + const status = parseMatrixQaCliJson(statusResult) as MatrixQaCliVerificationStatus; + assertMatrixQaCliAccountAddBootstrapStatus({ + expectedBackupVersion: added.verificationBootstrap.backupVersion, + expectedUserId: account.userId, + status, + }); + const cliDeviceId = status.deviceId ?? null; + + return { + artifacts: { + accountId, + backupVersion: added.verificationBootstrap.backupVersion ?? null, + cliDeviceId, + encryptionEnabled: added.encryptionEnabled, + verificationBootstrapAttempted: added.verificationBootstrap.attempted, + verificationBootstrapSuccess: added.verificationBootstrap.success, + }, + details: [ + "Matrix CLI account add --enable-e2ee created an encrypted account and bootstrapped recovery state", + `account add stdout: ${addArtifacts.stdoutPath}`, + `account add stderr: ${addArtifacts.stderrPath}`, + `verify status stdout: ${statusArtifacts.stdoutPath}`, + `verify status stderr: ${statusArtifacts.stderrPath}`, + `cli device: ${cliDeviceId ?? ""}`, + `cli verified by owner: ${status.verified ? "yes" : "no"}`, + `cli backup usable: ${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}`, + ].join("\n"), + }; + } finally { + await cli.dispose(); + } +} + +export async function runMatrixQaE2eeCliEncryptionSetupScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli-encryption-setup"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Encryption Setup Owner", + scenarioId: "matrix-e2ee-cli-encryption-setup", + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const cliDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA CLI Encryption Setup Device", + password: account.password, + userId: account.userId, + }); + if (!cliDevice.deviceId) { + throw new Error("Matrix E2EE CLI encryption setup login did not return a device id"); + } + const cli = await createMatrixQaCliE2eeSetupRuntime({ + artifactLabel: "cli-encryption-setup", + context, + initialConfig: buildMatrixQaCliE2eeAccountConfig({ + accountId, + accessToken: cliDevice.accessToken, + baseUrl: context.baseUrl, + deviceId: cliDevice.deviceId, + encryption: false, + name: "Matrix QA CLI Encryption Setup", + password: account.password, + userId: cliDevice.userId, + }), + }); + try { + const setupResult = await cli.run([ + "matrix", + "encryption", + "setup", + "--account", + accountId, + "--json", + ]); + const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "encryption-setup", + result: setupResult, + rootDir: cli.rootDir, + }); + const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; + if ( + setup.accountId !== accountId || + setup.success !== true || + setup.encryptionChanged !== true || + setup.bootstrap?.success !== true || + !setup.status + ) { + throw new Error( + `Matrix CLI encryption setup did not report a successful E2EE upgrade: ${setup.bootstrap?.error ?? "unknown error"}`, + ); + } + assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup", setup.status); + + const statusResult = await cli.run([ + "matrix", + "verify", + "status", + "--account", + accountId, + "--json", + ]); + const statusArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "verify-status", + result: statusResult, + rootDir: cli.rootDir, + }); + const status = parseMatrixQaCliJson(statusResult) as MatrixQaCliVerificationStatus; + assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup status", status); + + return { + artifacts: { + accountId, + cliDeviceId: status.deviceId ?? cliDevice.deviceId, + encryptionChanged: setup.encryptionChanged, + setupSuccess: setup.success, + verificationBootstrapSuccess: setup.bootstrap.success, + }, + details: [ + "Matrix CLI encryption setup upgraded an existing account and bootstrapped verification", + `encryption setup stdout: ${setupArtifacts.stdoutPath}`, + `encryption setup stderr: ${setupArtifacts.stderrPath}`, + `verify status stdout: ${statusArtifacts.stdoutPath}`, + `verify status stderr: ${statusArtifacts.stderrPath}`, + `cli device: ${status.deviceId ?? cliDevice.deviceId}`, + `cli verified by owner: ${status.verified ? "yes" : "no"}`, + `cli backup usable: ${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}`, + ].join("\n"), + }; + } finally { + await cli.dispose(); + } +} + +export async function runMatrixQaE2eeCliEncryptionSetupIdempotentScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli-encryption-idempotent"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Encryption Idempotent Owner", + scenarioId: "matrix-e2ee-cli-encryption-setup-idempotent", + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const cliDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA CLI Encryption Idempotent Device", + password: account.password, + userId: account.userId, + }); + if (!cliDevice.deviceId) { + throw new Error("Matrix E2EE CLI idempotent setup login did not return a device id"); + } + const cli = await createMatrixQaCliE2eeSetupRuntime({ + artifactLabel: "cli-encryption-setup-idempotent", + context, + initialConfig: buildMatrixQaCliE2eeAccountConfig({ + accountId, + accessToken: cliDevice.accessToken, + baseUrl: context.baseUrl, + deviceId: cliDevice.deviceId, + encryption: true, + name: "Matrix QA CLI Encryption Setup Idempotent", + password: account.password, + userId: cliDevice.userId, + }), + }); + try { + const setupArgs = ["matrix", "encryption", "setup", "--account", accountId, "--json"]; + const firstResult = await cli.run(setupArgs); + const firstArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "encryption-setup-first", + result: firstResult, + rootDir: cli.rootDir, + }); + const first = parseMatrixQaCliJson(firstResult) as MatrixQaCliEncryptionSetupStatus; + if ( + first.accountId !== accountId || + first.success !== true || + first.encryptionChanged !== false || + first.bootstrap?.success !== true || + !first.status + ) { + throw new Error( + `Matrix CLI encryption setup was not idempotent on first run: ${first.bootstrap?.error ?? "unknown error"}`, + ); + } + assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup idempotent first run", first.status); + + const secondResult = await cli.run(setupArgs); + const secondArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "encryption-setup-second", + result: secondResult, + rootDir: cli.rootDir, + }); + const second = parseMatrixQaCliJson(secondResult) as MatrixQaCliEncryptionSetupStatus; + if ( + second.accountId !== accountId || + second.success !== true || + second.encryptionChanged !== false || + second.bootstrap?.success !== true || + !second.status + ) { + throw new Error( + `Matrix CLI encryption setup was not idempotent on second run: ${second.bootstrap?.error ?? "unknown error"}`, + ); + } + assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup idempotent second run", second.status); + + return { + artifacts: { + accountId, + cliDeviceId: second.status.deviceId ?? cliDevice.deviceId, + firstEncryptionChanged: first.encryptionChanged, + secondEncryptionChanged: second.encryptionChanged, + setupSuccess: second.success, + verificationBootstrapSuccess: second.bootstrap.success, + }, + details: [ + "Matrix CLI encryption setup stayed idempotent on an already encrypted account", + `first setup stdout: ${firstArtifacts.stdoutPath}`, + `first setup stderr: ${firstArtifacts.stderrPath}`, + `second setup stdout: ${secondArtifacts.stdoutPath}`, + `second setup stderr: ${secondArtifacts.stderrPath}`, + `cli device: ${second.status.deviceId ?? cliDevice.deviceId}`, + `first encryption changed: ${first.encryptionChanged ? "yes" : "no"}`, + `second encryption changed: ${second.encryptionChanged ? "yes" : "no"}`, + ].join("\n"), + }; + } finally { + await cli.dispose(); + } +} + +export async function runMatrixQaE2eeCliEncryptionSetupBootstrapFailureScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli-encryption-failure"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Encryption Failure Owner", + scenarioId: "matrix-e2ee-cli-encryption-setup-bootstrap-failure", + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const cliDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA CLI Encryption Failure Device", + password: account.password, + userId: account.userId, + }); + if (!cliDevice.deviceId) { + throw new Error("Matrix E2EE CLI bootstrap-failure login did not return a device id"); + } + const proxy = await startMatrixQaFaultProxy({ + targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl, + ...context.faultProxyObserver, + rules: [buildRoomKeyBackupUnavailableFaultRule(cliDevice.accessToken)], + }); + const cli = await createMatrixQaCliE2eeSetupRuntime({ + artifactLabel: "cli-encryption-setup-bootstrap-failure", + context, + initialConfig: buildMatrixQaCliE2eeAccountConfig({ + accountId, + accessToken: cliDevice.accessToken, + baseUrl: proxy.baseUrl, + deviceId: cliDevice.deviceId, + encryption: false, + name: "Matrix QA CLI Encryption Setup Bootstrap Failure", + password: account.password, + userId: cliDevice.userId, + }), + }); + try { + const failed = await runMatrixQaCliExpectedFailure({ + args: ["matrix", "encryption", "setup", "--account", accountId, "--json"], + start: cli.start, + timeoutMs: context.timeoutMs, + }); + const artifacts = await writeMatrixQaCliOutputArtifacts({ + label: "encryption-setup-bootstrap-failure", + result: failed, + rootDir: cli.rootDir, + }); + const payload = parseMatrixQaCliJson(failed) as MatrixQaCliEncryptionSetupStatus; + if (payload.success !== false && payload.bootstrap?.success !== false) { + throw new Error("Matrix CLI encryption setup failure did not report unsuccessful bootstrap"); + } + const faultHits = proxy.hits(); + if (faultHits.length === 0) { + throw new Error("Matrix CLI encryption setup bootstrap-failure proxy was not exercised"); + } + const bootstrapError = payload.bootstrap?.error ?? ""; + if (!bootstrapError.toLowerCase().includes("room key backup")) { + throw new Error( + `Matrix CLI encryption setup failed for an unexpected reason: ${bootstrapError}`, + ); + } + + return { + artifacts: { + accountId, + bootstrapErrorPreview: truncateUtf16Safe(bootstrapError, 240), + bootstrapSuccess: false, + cliDeviceId: cliDevice.deviceId, + faultedEndpoint: faultHits[0]?.path, + faultHitCount: faultHits.length, + faultRuleId: MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID, + }, + details: [ + "Matrix CLI encryption setup surfaced a bootstrap failure from a faulted room-key backup endpoint", + `failure stdout: ${artifacts.stdoutPath}`, + `failure stderr: ${artifacts.stderrPath}`, + `fault hits: ${faultHits.length}`, + `fault endpoint: ${faultHits[0]?.path ?? ""}`, + `bootstrap error: ${bootstrapError}`, + ].join("\n"), + }; + } finally { + await Promise.all([cli.dispose(), proxy.stop().catch(() => undefined)]); + } +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-config.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-config.ts new file mode 100644 index 000000000000..6da423e0bc7f --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-config.ts @@ -0,0 +1,162 @@ +// Qa Matrix plugin module implements CLI config assertions for E2EE scenarios. +import { readFile } from "node:fs/promises"; +import { + formatMatrixQaCliCommand, + redactMatrixQaCliOutput, + type MatrixQaCliRunResult, + type MatrixQaCliSession, +} from "./scenario-runtime-cli.js"; +import { + buildMatrixQaPluginActivationConfig, + isMatrixQaCliBackupUsable, + type MatrixQaCliVerificationStatus, +} from "./scenario-runtime-e2ee-cli-shared.js"; + +export function assertMatrixQaCliE2eeStatus( + label: string, + status: MatrixQaCliVerificationStatus, + opts: { allowUntrustedMatchingKey?: boolean } = {}, +) { + if ( + status.verified !== true || + status.crossSigningVerified !== true || + status.signedByOwner !== true || + !isMatrixQaCliBackupUsable(status.backup, opts) + ) { + throw new Error( + `${label} did not leave the CLI account fully verified and backup-usable: ownerVerified=${ + status.verified === true && + status.crossSigningVerified === true && + status.signedByOwner === true + ? "yes" + : "no" + }, backupUsable=${isMatrixQaCliBackupUsable(status.backup, opts) ? "yes" : "no"}${ + status.backup?.keyLoadError ? `, backupError=${status.backup.keyLoadError}` : "" + }`, + ); + } +} + +export function assertMatrixQaCliAccountAddBootstrapStatus(params: { + expectedBackupVersion?: string | null; + expectedUserId: string; + status: MatrixQaCliVerificationStatus; +}) { + const { expectedBackupVersion, expectedUserId, status } = params; + if (status.encryptionEnabled !== true || status.recoveryKeyStored !== true) { + throw new Error( + "Matrix CLI account add --enable-e2ee degraded status did not keep encryption and recovery key state", + ); + } + if (status.userId !== expectedUserId) { + throw new Error( + `Matrix CLI account add --enable-e2ee status user mismatch: expected ${expectedUserId}, got ${status.userId ?? ""}`, + ); + } + if (!status.deviceId || status.serverDeviceKnown !== true) { + throw new Error( + "Matrix CLI account add --enable-e2ee degraded status did not resolve the current server-known device", + ); + } + if (expectedBackupVersion && status.backupVersion !== expectedBackupVersion) { + throw new Error( + `Matrix CLI account add --enable-e2ee backup version mismatch: expected ${expectedBackupVersion}, got ${status.backupVersion ?? ""}`, + ); + } + if (status.backup?.keyLoadError) { + throw new Error( + `Matrix CLI account add --enable-e2ee degraded status reported backup key error: ${status.backup.keyLoadError}`, + ); + } +} + +export async function runMatrixQaCliExpectedFailure(params: { + args: string[]; + start: (args: string[], timeoutMs?: number) => MatrixQaCliSession; + stdin?: string; + timeoutMs: number; +}): Promise { + const session = params.start(params.args, params.timeoutMs); + try { + if (params.stdin !== undefined) { + await session.writeStdin(params.stdin); + session.endStdin(); + } + const result = await session.wait(); + throw new Error( + `${formatMatrixQaCliCommand(params.args)} unexpectedly succeeded with stdout:\n${redactMatrixQaCliOutput( + result.stdout, + )}`, + ); + } catch (error) { + if (error instanceof Error && error.message.includes("unexpectedly succeeded")) { + throw error; + } + const output = session.output(); + if (!output.stdout.trim() && !output.stderr.trim()) { + throw error; + } + return { + args: params.args, + exitCode: 1, + stderr: output.stderr, + stdout: output.stdout, + }; + } finally { + session.kill(); + } +} + +export function buildMatrixQaCliE2eeAccountConfig(params: { + accountId: string; + accessToken: string; + baseUrl: string; + deviceId: string; + encryption: boolean; + name: string; + password?: string; + userId: string; +}) { + return { + ...buildMatrixQaPluginActivationConfig(), + channels: { + matrix: { + defaultAccount: params.accountId, + accounts: { + [params.accountId]: { + accessToken: params.accessToken, + deviceId: params.deviceId, + encryption: params.encryption, + homeserver: params.baseUrl, + initialSyncLimit: 1, + name: params.name, + network: { + dangerouslyAllowPrivateNetwork: true, + }, + ...(params.password ? { password: params.password } : {}), + startupVerification: "off", + userId: params.userId, + }, + }, + }, + }, + }; +} + +export async function readMatrixQaCliConfig(pathname: string): Promise<{ + channels?: { + matrix?: { + accounts?: Record>; + defaultAccount?: string; + }; + }; +}> { + return JSON.parse(await readFile(pathname, "utf8")) as { + channels?: { + matrix?: { + accounts?: Record>; + defaultAccount?: string; + }; + }; + }; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-gateway.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-gateway.ts new file mode 100644 index 000000000000..6f392ffd8477 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-gateway.ts @@ -0,0 +1,384 @@ +// Qa Matrix plugin module implements gateway CLI E2EE scenarios. +import { randomUUID } from "node:crypto"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { createMatrixQaE2eeScenarioClient } from "../../substrate/e2ee-client.js"; +import { buildMatrixQaE2eeScenarioRoomKey } from "./scenario-catalog.js"; +import { + patchMatrixQaGatewayMatrixAccount, + replaceMatrixQaGatewayMatrixAccount, +} from "./scenario-runtime-config.js"; +import { + assertMatrixQaCliE2eeStatus, + readMatrixQaCliConfig, +} from "./scenario-runtime-e2ee-cli-config.js"; +import { + createMatrixQaCliE2eeSetupRuntime, + createMatrixQaCliGatewayRuntime, +} from "./scenario-runtime-e2ee-cli-runtime.js"; +import { + buildMatrixQaPluginActivationConfig, + parseMatrixQaCliJson, + registerMatrixQaCliE2eeAccount, + type MatrixQaCliEncryptionSetupStatus, + writeMatrixQaCliOutputArtifacts, +} from "./scenario-runtime-e2ee-cli-shared.js"; +import { buildMatrixE2eeReplyArtifact } from "./scenario-runtime-e2ee-room.js"; +import { + ensureMatrixQaE2eeOwnDeviceVerified, + requireMatrixQaE2eeOutputDir, + requireMatrixQaGatewayConfigPath, +} from "./scenario-runtime-e2ee-shared.js"; +import { + assertTopLevelReplyArtifact, + buildMatrixQaToken, + buildMatrixReplyDetails, + buildMentionPrompt, + isMatrixQaExactMarkerReply, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runMatrixQaE2eeCliEncryptionSetupMultiAccountScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli-multi-target"; + const decoyAccountId = "cli-multi-decoy"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Multi Account Owner", + scenarioId: "matrix-e2ee-cli-encryption-setup-multi-account", + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const cliDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA CLI Multi Account Target Device", + password: account.password, + userId: account.userId, + }); + if (!cliDevice.deviceId) { + throw new Error("Matrix E2EE CLI multi-account setup login did not return a device id"); + } + const cli = await createMatrixQaCliE2eeSetupRuntime({ + artifactLabel: "cli-encryption-setup-multi-account", + context, + initialConfig: { + ...buildMatrixQaPluginActivationConfig(), + channels: { + matrix: { + defaultAccount: decoyAccountId, + accounts: { + [decoyAccountId]: { + accessToken: "decoy-token", + deviceId: "DECOYDEVICE", + encryption: false, + homeserver: context.baseUrl, + initialSyncLimit: 1, + name: "Matrix QA CLI Multi Account Decoy", + startupVerification: "off", + userId: "@decoy:matrix-qa.test", + }, + [accountId]: { + accessToken: cliDevice.accessToken, + deviceId: cliDevice.deviceId, + encryption: false, + homeserver: context.baseUrl, + initialSyncLimit: 1, + name: "Matrix QA CLI Multi Account Target", + network: { + dangerouslyAllowPrivateNetwork: true, + }, + password: account.password, + startupVerification: "off", + userId: cliDevice.userId, + }, + }, + }, + }, + }, + }); + try { + const setupResult = await cli.run([ + "matrix", + "encryption", + "setup", + "--account", + accountId, + "--json", + ]); + const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "encryption-setup-multi-account", + result: setupResult, + rootDir: cli.rootDir, + }); + const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; + if ( + setup.accountId !== accountId || + setup.success !== true || + setup.encryptionChanged !== true || + setup.bootstrap?.success !== true || + !setup.status + ) { + throw new Error( + `Matrix CLI multi-account encryption setup did not target the requested account: ${setup.bootstrap?.error ?? "unknown error"}`, + ); + } + assertMatrixQaCliE2eeStatus("Matrix CLI multi-account encryption setup", setup.status); + + const config = await readMatrixQaCliConfig(cli.configPath); + const matrix = config.channels?.matrix; + const target = matrix?.accounts?.[accountId]; + const decoy = matrix?.accounts?.[decoyAccountId]; + const defaultAccountPreserved = matrix?.defaultAccount === decoyAccountId; + const decoyAccountPreserved = + decoy?.encryption === false && + decoy?.accessToken === "decoy-token" && + decoy?.deviceId === "DECOYDEVICE"; + if (!defaultAccountPreserved) { + throw new Error("Matrix CLI multi-account setup changed the default account"); + } + if (!decoyAccountPreserved) { + throw new Error("Matrix CLI multi-account setup mutated the decoy account"); + } + if (target?.encryption !== true) { + throw new Error("Matrix CLI multi-account setup did not enable encryption on the target"); + } + + return { + artifacts: { + accountId, + cliDeviceId: setup.status.deviceId ?? cliDevice.deviceId, + decoyAccountPreserved, + defaultAccountPreserved, + encryptionChanged: setup.encryptionChanged, + setupSuccess: setup.success, + verificationBootstrapSuccess: setup.bootstrap.success, + }, + details: [ + "Matrix CLI encryption setup changed only the requested account in a multi-account config", + `setup stdout: ${setupArtifacts.stdoutPath}`, + `setup stderr: ${setupArtifacts.stderrPath}`, + `default account preserved: ${defaultAccountPreserved ? "yes" : "no"}`, + `decoy account preserved: ${decoyAccountPreserved ? "yes" : "no"}`, + `cli device: ${setup.status.deviceId ?? cliDevice.deviceId}`, + ].join("\n"), + }; + } finally { + await cli.dispose(); + } +} + +export async function runMatrixQaE2eeCliSetupThenGatewayReplyScenario( + context: MatrixQaScenarioContext, +): Promise { + if (!context.restartGatewayAfterStateMutation) { + throw new Error( + "Matrix CLI setup gateway reply scenario requires hard gateway restart support", + ); + } + const gatewayConfigPath = requireMatrixQaGatewayConfigPath(context); + const accountId = "cli-setup-gateway"; + const scenarioId = "matrix-e2ee-cli-setup-then-gateway-reply"; + const roomKey = buildMatrixQaE2eeScenarioRoomKey(scenarioId); + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Setup Gateway", + scenarioId, + }); + const driverAccount = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Setup Driver", + scenarioId, + }); + const driverApi = createMatrixQaClient({ + accessToken: driverAccount.accessToken, + baseUrl: context.baseUrl, + }); + const gatewayApi = createMatrixQaClient({ + accessToken: account.accessToken, + baseUrl: context.baseUrl, + }); + const roomId = await driverApi.createPrivateRoom({ + encrypted: true, + inviteUserIds: [account.userId], + name: "Matrix QA CLI Setup Gateway E2EE", + }); + await gatewayApi.joinRoom(roomId); + + const accountConfig = { + accessToken: account.accessToken, + deviceId: account.deviceId, + dm: { + allowFrom: [driverAccount.userId], + enabled: true, + policy: "allowlist", + sessionScope: "per-room", + threadReplies: "inbound", + }, + enabled: true, + encryption: false, + groupAllowFrom: [driverAccount.userId], + groupPolicy: "allowlist", + groups: { + [roomId]: { + enabled: true, + requireMention: true, + }, + }, + homeserver: context.baseUrl, + initialSyncLimit: 1, + name: "Matrix QA CLI Setup Gateway", + network: { + dangerouslyAllowPrivateNetwork: true, + }, + password: account.password, + startupVerification: "off", + threadReplies: "inbound", + userId: account.userId, + }; + await context.restartGatewayAfterStateMutation( + async () => { + await replaceMatrixQaGatewayMatrixAccount({ + accountConfig, + accountId, + configPath: gatewayConfigPath, + }); + }, + { + timeoutMs: context.timeoutMs, + waitAccountId: accountId, + }, + ); + await context.waitGatewayAccountReady?.(accountId, { + timeoutMs: context.timeoutMs, + }); + const cli = await createMatrixQaCliGatewayRuntime({ + artifactLabel: "cli-setup-then-gateway-reply", + context, + }); + try { + const setupResult = await cli.run([ + "matrix", + "encryption", + "setup", + "--account", + accountId, + "--json", + ]); + const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "encryption-setup", + result: setupResult, + rootDir: cli.rootDir, + }); + const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; + if ( + setup.accountId !== accountId || + setup.success !== true || + setup.bootstrap?.success !== true + ) { + throw new Error( + `Matrix CLI gateway account setup did not succeed: ${setup.bootstrap?.error ?? "unknown error"}`, + ); + } + if (setup.status) { + assertMatrixQaCliE2eeStatus("Matrix CLI gateway account setup", setup.status); + } + await context.restartGatewayAfterStateMutation( + async () => { + await patchMatrixQaGatewayMatrixAccount({ + accountPatch: { + encryption: true, + password: account.password, + }, + accountId, + configPath: gatewayConfigPath, + }); + }, + { + timeoutMs: context.timeoutMs, + waitAccountId: accountId, + }, + ); + await context.waitGatewayAccountReady?.(accountId, { + timeoutMs: context.timeoutMs, + }); + const driverClient = await createMatrixQaE2eeScenarioClient({ + accessToken: driverAccount.accessToken, + actorId: `driver-cli-setup-gateway-${randomUUID().slice(0, 8)}`, + baseUrl: context.baseUrl, + deviceId: driverAccount.deviceId, + observedEvents: context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(context), + password: driverAccount.password, + scenarioId, + timeoutMs: context.timeoutMs, + userId: driverAccount.userId, + }); + const replied = await (async () => { + try { + await ensureMatrixQaE2eeOwnDeviceVerified({ + client: driverClient, + label: "Matrix CLI setup scenario driver", + }); + await driverClient.waitForJoinedMember({ + roomId, + timeoutMs: context.timeoutMs, + userId: account.userId, + }); + await driverClient.prime(); + const token = buildMatrixQaToken("MATRIX_QA_E2EE_CLI_GATEWAY"); + const driverEventId = await driverClient.sendTextMessage({ + body: buildMentionPrompt(account.userId, token), + mentionUserIds: [account.userId], + roomId, + }); + const matched = await driverClient.waitForRoomEvent({ + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId, + sutUserId: account.userId, + token, + }) && event.relatesTo === undefined, + roomId, + timeoutMs: context.timeoutMs, + }); + const reply = buildMatrixE2eeReplyArtifact(matched.event, token); + assertTopLevelReplyArtifact("gateway reply", reply); + return { + driverEventId, + reply, + }; + } finally { + await driverClient.stop(); + } + })(); + + return { + artifacts: { + accountId, + cliDeviceId: setup.status?.deviceId ?? account.deviceId ?? null, + driverUserId: driverAccount.userId, + encryptionChanged: setup.encryptionChanged, + gatewayReply: replied.reply, + gatewayUserId: account.userId, + roomKey, + roomId, + setupSuccess: setup.success, + verificationBootstrapSuccess: setup.bootstrap.success, + }, + details: [ + "Matrix CLI encryption setup left the gateway able to reply in an encrypted room", + `setup stdout: ${setupArtifacts.stdoutPath}`, + `setup stderr: ${setupArtifacts.stderrPath}`, + `driver user: ${driverAccount.userId}`, + `gateway user: ${account.userId}`, + `encrypted room key: ${roomKey}`, + `encrypted room id: ${roomId}`, + `driver event: ${replied.driverEventId}`, + ...buildMatrixReplyDetails("gateway reply", replied.reply), + ].join("\n"), + }; + } finally { + await cli.dispose(); + } +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-recovery.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-recovery.ts new file mode 100644 index 000000000000..178a2ae85755 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-recovery.ts @@ -0,0 +1,246 @@ +// Qa Matrix plugin module implements recovery-key CLI E2EE scenarios. +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { + assertMatrixQaCliE2eeStatus, + buildMatrixQaCliE2eeAccountConfig, + runMatrixQaCliExpectedFailure, +} from "./scenario-runtime-e2ee-cli-config.js"; +import { createMatrixQaCliE2eeSetupRuntime } from "./scenario-runtime-e2ee-cli-runtime.js"; +import { + createMatrixQaE2eeCliOwnerClient, + isMatrixQaCliBackupUsable, + parseMatrixQaCliJson, + registerMatrixQaCliE2eeAccount, + type MatrixQaCliEncryptionSetupStatus, + writeMatrixQaCliOutputArtifacts, +} from "./scenario-runtime-e2ee-cli-shared.js"; +import { ensureMatrixQaE2eeOwnDeviceVerified } from "./scenario-runtime-e2ee-shared.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runMatrixQaE2eeCliRecoveryKeySetupScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli-recovery-key-setup"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Recovery Key Owner", + scenarioId: "matrix-e2ee-cli-recovery-key-setup", + }); + const owner = await createMatrixQaE2eeCliOwnerClient({ + account, + context, + scenarioId: "matrix-e2ee-cli-recovery-key-setup", + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ + client: owner, + label: "driver", + }); + const encodedRecoveryKey = ready.recoveryKey?.encodedPrivateKey?.trim(); + if (!encodedRecoveryKey) { + await owner.stop().catch(() => undefined); + throw new Error("Matrix E2EE CLI recovery-key setup did not expose a recovery key"); + } + const cliDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA CLI Recovery Key Setup Device", + password: account.password, + userId: account.userId, + }); + if (!cliDevice.deviceId) { + await owner.stop().catch(() => undefined); + throw new Error("Matrix E2EE CLI recovery-key setup login did not return a device id"); + } + const cli = await createMatrixQaCliE2eeSetupRuntime({ + artifactLabel: "cli-recovery-key-setup", + context, + initialConfig: buildMatrixQaCliE2eeAccountConfig({ + accountId, + accessToken: cliDevice.accessToken, + baseUrl: context.baseUrl, + deviceId: cliDevice.deviceId, + encryption: false, + name: "Matrix QA CLI Recovery Key Setup", + password: account.password, + userId: cliDevice.userId, + }), + }); + try { + const setupResult = await cli.run( + ["matrix", "encryption", "setup", "--account", accountId, "--recovery-key-stdin", "--json"], + context.timeoutMs, + `${encodedRecoveryKey}\n`, + ); + const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "recovery-key-setup", + result: setupResult, + rootDir: cli.rootDir, + }); + const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; + if ( + setup.accountId !== accountId || + setup.success !== true || + setup.encryptionChanged !== true || + setup.bootstrap?.success !== true || + !setup.status + ) { + throw new Error( + `Matrix CLI recovery-key encryption setup did not succeed: ${setup.bootstrap?.error ?? "unknown error"}`, + ); + } + assertMatrixQaCliE2eeStatus("Matrix CLI recovery-key encryption setup", setup.status, { + allowUntrustedMatchingKey: true, + }); + + return { + artifacts: { + accountId, + backupVersion: setup.status.backupVersion ?? ready.verification.backupVersion ?? null, + cliDeviceId: setup.status.deviceId ?? cliDevice.deviceId, + encryptionChanged: setup.encryptionChanged, + recoveryKeyId: ready.recoveryKey?.keyId ?? null, + recoveryKeyStored: true, + setupSuccess: setup.success, + verificationBootstrapSuccess: setup.bootstrap.success, + }, + details: [ + "Matrix CLI encryption setup accepted a recovery key on a second device", + `recovery setup stdout: ${setupArtifacts.stdoutPath}`, + `recovery setup stderr: ${setupArtifacts.stderrPath}`, + `owner backup version: ${ready.verification.backupVersion ?? ""}`, + `recovery key id: ${ready.recoveryKey?.keyId ?? ""}`, + `cli device: ${setup.status.deviceId ?? cliDevice.deviceId}`, + `cli verified by owner: ${setup.status.verified ? "yes" : "no"}`, + `cli backup usable: ${ + isMatrixQaCliBackupUsable(setup.status.backup, { allowUntrustedMatchingKey: true }) + ? "yes" + : "no" + }`, + ].join("\n"), + }; + } finally { + try { + await owner.stop().catch(() => undefined); + await owner.deleteOwnDevices([cliDevice.deviceId]).catch(() => undefined); + } finally { + await cli.dispose(); + } + } +} + +export async function runMatrixQaE2eeCliRecoveryKeyInvalidScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli-invalid-recovery-key"; + const invalidRecoveryKey = "not-a-valid-matrix-recovery-key"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Invalid Recovery Key Owner", + scenarioId: "matrix-e2ee-cli-recovery-key-invalid", + }); + const owner = await createMatrixQaE2eeCliOwnerClient({ + account, + context, + scenarioId: "matrix-e2ee-cli-recovery-key-invalid", + }); + const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ + client: owner, + label: "cli invalid recovery-key owner", + }); + if (!ready.recoveryKey?.encodedPrivateKey?.trim()) { + await owner.stop().catch(() => undefined); + throw new Error("Matrix E2EE CLI invalid recovery-key setup did not seed secret storage"); + } + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const cliDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA CLI Invalid Recovery Key Device", + password: account.password, + userId: account.userId, + }); + if (!cliDevice.deviceId) { + await owner.stop().catch(() => undefined); + throw new Error("Matrix E2EE CLI invalid recovery-key login did not return a device id"); + } + const cli = await createMatrixQaCliE2eeSetupRuntime({ + artifactLabel: "cli-recovery-key-invalid", + context, + initialConfig: buildMatrixQaCliE2eeAccountConfig({ + accountId, + accessToken: cliDevice.accessToken, + baseUrl: context.baseUrl, + deviceId: cliDevice.deviceId, + encryption: false, + name: "Matrix QA CLI Invalid Recovery Key", + password: account.password, + userId: cliDevice.userId, + }), + }); + try { + const failed = await runMatrixQaCliExpectedFailure({ + args: [ + "matrix", + "encryption", + "setup", + "--account", + accountId, + "--recovery-key-stdin", + "--json", + ], + start: cli.start, + stdin: `${invalidRecoveryKey}\n`, + timeoutMs: context.timeoutMs, + }); + const artifacts = await writeMatrixQaCliOutputArtifacts({ + label: "recovery-key-invalid", + result: failed, + rootDir: cli.rootDir, + }); + const payload = parseMatrixQaCliJson(failed) as MatrixQaCliEncryptionSetupStatus & { + error?: string; + }; + if (payload.success !== false && payload.bootstrap?.success !== false) { + throw new Error("Matrix CLI invalid recovery-key setup did not report failure"); + } + const failure = payload.bootstrap?.error ?? payload.error ?? ""; + if (!/recovery|secret|key/i.test(failure)) { + throw new Error( + `Matrix CLI invalid recovery-key setup failed for an unexpected reason: ${failure}`, + ); + } + if (failed.stdout.includes(invalidRecoveryKey) || failed.stderr.includes(invalidRecoveryKey)) { + throw new Error("Matrix CLI invalid recovery-key output leaked the recovery key"); + } + + return { + artifacts: { + accountId, + bootstrapErrorPreview: truncateUtf16Safe(failure, 240), + bootstrapSuccess: false, + cliDeviceId: cliDevice.deviceId, + encryptionChanged: payload.encryptionChanged, + recoveryKeyAccepted: false, + recoveryKeyRejected: true, + setupSuccess: false, + }, + details: [ + "Matrix CLI encryption setup rejected an invalid recovery key without leaking it", + `failure stdout: ${artifacts.stdoutPath}`, + `failure stderr: ${artifacts.stderrPath}`, + `cli device: ${cliDevice.deviceId}`, + `failure: ${failure}`, + ].join("\n"), + }; + } finally { + try { + await owner.stop().catch(() => undefined); + await owner.deleteOwnDevices([cliDevice.deviceId]).catch(() => undefined); + } finally { + await cli.dispose(); + } + } +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-runtime.ts new file mode 100644 index 000000000000..f21ecccace77 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-runtime.ts @@ -0,0 +1,206 @@ +// Qa Matrix plugin module implements CLI runtime setup for E2EE scenarios. +import { randomUUID } from "node:crypto"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { runMatrixQaOpenClawCli, startMatrixQaOpenClawCli } from "./scenario-runtime-cli.js"; +import { + assertMatrixQaPrivatePathMode, + buildMatrixQaEmptyMatrixCliConfig, +} from "./scenario-runtime-e2ee-cli-shared.js"; +import { + requireMatrixQaCliRuntimeEnv, + requireMatrixQaE2eeOutputDir, +} from "./scenario-runtime-e2ee-shared.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; + +export async function createMatrixQaCliSelfVerificationRuntime(params: { + accountId: string; + accessToken: string; + context: MatrixQaScenarioContext; + deviceId: string; + userId: string; +}) { + const outputDir = requireMatrixQaE2eeOutputDir(params.context); + const rootDir = await mkdtemp( + path.join(resolvePreferredOpenClawTmpDir(), "openclaw-matrix-cli-qa-"), + ); + const artifactDir = path.join( + outputDir, + "cli-self-verification", + randomUUID().replaceAll("-", "").slice(0, 12), + ); + const stateDir = path.join(rootDir, "state"); + const configPath = path.join(rootDir, "config.json"); + await chmod(rootDir, 0o700).catch(() => undefined); + await assertMatrixQaPrivatePathMode(rootDir, "Matrix QA CLI temp directory"); + await mkdir(artifactDir, { mode: 0o700, recursive: true }); + await chmod(artifactDir, 0o700).catch(() => undefined); + await assertMatrixQaPrivatePathMode(artifactDir, "Matrix QA CLI artifact directory"); + await mkdir(stateDir, { mode: 0o700, recursive: true }); + await chmod(stateDir, 0o700).catch(() => undefined); + await assertMatrixQaPrivatePathMode(stateDir, "Matrix QA CLI state directory"); + await writeFile( + configPath, + `${JSON.stringify( + { + plugins: { + allow: ["matrix"], + entries: { + matrix: { enabled: true }, + }, + }, + channels: { + matrix: { + defaultAccount: params.accountId, + accounts: { + [params.accountId]: { + accessToken: params.accessToken, + deviceId: params.deviceId, + encryption: true, + homeserver: params.context.baseUrl, + initialSyncLimit: 0, + name: "Matrix QA CLI self-verification", + network: { + dangerouslyAllowPrivateNetwork: true, + }, + startupVerification: "off", + userId: params.userId, + }, + }, + }, + }, + }, + null, + 2, + )}\n`, + { flag: "wx", mode: 0o600 }, + ); + await assertMatrixQaPrivatePathMode(configPath, "Matrix QA CLI config file"); + const env = { + ...requireMatrixQaCliRuntimeEnv(params.context), + FORCE_COLOR: "0", + NO_COLOR: "1", + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_DISABLE_AUTO_UPDATE: "1", + OPENCLAW_STATE_DIR: stateDir, + }; + const run = async (args: string[], timeoutMs = params.context.timeoutMs, stdin?: string) => + await runMatrixQaOpenClawCli({ + args, + env, + stdin, + timeoutMs, + }); + const start = (args: string[], timeoutMs = params.context.timeoutMs) => + startMatrixQaOpenClawCli({ + args, + env, + timeoutMs, + }); + return { + configPath, + dispose: async () => { + await rm(rootDir, { force: true, recursive: true }); + }, + run, + rootDir: artifactDir, + start, + stateDir, + }; +} + +export async function createMatrixQaCliE2eeSetupRuntime(params: { + artifactLabel: string; + context: MatrixQaScenarioContext; + initialConfig?: Record; +}) { + const outputDir = requireMatrixQaE2eeOutputDir(params.context); + const rootDir = await mkdtemp( + path.join(resolvePreferredOpenClawTmpDir(), "openclaw-matrix-e2ee-setup-qa-"), + ); + const artifactDir = path.join( + outputDir, + params.artifactLabel, + randomUUID().replaceAll("-", "").slice(0, 12), + ); + const stateDir = path.join(rootDir, "state"); + const configPath = path.join(rootDir, "config.json"); + await chmod(rootDir, 0o700).catch(() => undefined); + await assertMatrixQaPrivatePathMode(rootDir, "Matrix QA CLI temp directory"); + await mkdir(artifactDir, { mode: 0o700, recursive: true }); + await chmod(artifactDir, 0o700).catch(() => undefined); + await assertMatrixQaPrivatePathMode(artifactDir, "Matrix QA CLI artifact directory"); + await mkdir(stateDir, { mode: 0o700, recursive: true }); + await chmod(stateDir, 0o700).catch(() => undefined); + await assertMatrixQaPrivatePathMode(stateDir, "Matrix QA CLI state directory"); + await writeFile( + configPath, + `${JSON.stringify(params.initialConfig ?? buildMatrixQaEmptyMatrixCliConfig(), null, 2)}\n`, + { flag: "wx", mode: 0o600 }, + ); + await assertMatrixQaPrivatePathMode(configPath, "Matrix QA CLI config file"); + const env = { + ...requireMatrixQaCliRuntimeEnv(params.context), + FORCE_COLOR: "0", + NO_COLOR: "1", + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_DISABLE_AUTO_UPDATE: "1", + OPENCLAW_STATE_DIR: stateDir, + }; + const run = async (args: string[], timeoutMs = params.context.timeoutMs, stdin?: string) => + await runMatrixQaOpenClawCli({ + args, + env, + stdin, + timeoutMs, + }); + const start = (args: string[], timeoutMs = params.context.timeoutMs) => + startMatrixQaOpenClawCli({ + args, + env, + timeoutMs, + }); + return { + configPath, + dispose: async () => { + await rm(rootDir, { force: true, recursive: true }); + }, + run, + rootDir: artifactDir, + start, + stateDir, + }; +} + +export async function createMatrixQaCliGatewayRuntime(params: { + artifactLabel: string; + context: MatrixQaScenarioContext; +}) { + const outputDir = requireMatrixQaE2eeOutputDir(params.context); + const artifactDir = path.join( + outputDir, + params.artifactLabel, + randomUUID().replaceAll("-", "").slice(0, 12), + ); + await mkdir(artifactDir, { mode: 0o700, recursive: true }); + await chmod(artifactDir, 0o700).catch(() => undefined); + await assertMatrixQaPrivatePathMode(artifactDir, "Matrix QA CLI artifact directory"); + const env = { + ...requireMatrixQaCliRuntimeEnv(params.context), + FORCE_COLOR: "0", + NO_COLOR: "1", + OPENCLAW_DISABLE_AUTO_UPDATE: "1", + }; + const run = async (args: string[], timeoutMs = params.context.timeoutMs) => + await runMatrixQaOpenClawCli({ + args, + env, + timeoutMs, + }); + return { + dispose: async () => undefined, + rootDir: artifactDir, + run, + }; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-shared.ts new file mode 100644 index 000000000000..57ee22d7967d --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-shared.ts @@ -0,0 +1,289 @@ +// Qa Matrix plugin module implements shared CLI scenario runtime E2EE behavior. +import { randomUUID } from "node:crypto"; +import { chmod, mkdir, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { MatrixVerificationSummary } from "@openclaw/matrix/test-api.js"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { createMatrixQaE2eeScenarioClient } from "../../substrate/e2ee-client.js"; +import type { MatrixQaE2eeScenarioId } from "./scenario-catalog.js"; +import { + formatMatrixQaCliCommand, + redactMatrixQaCliOutput, + type MatrixQaCliRunResult, +} from "./scenario-runtime-cli.js"; +import { + formatMatrixQaSasEmoji, + requireMatrixQaE2eeOutputDir, + requireMatrixQaRegistrationToken, +} from "./scenario-runtime-e2ee-shared.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; + +export type MatrixQaCliVerificationStatus = { + backup?: { + decryptionKeyCached?: boolean | null; + keyLoadError?: string | null; + matchesDecryptionKey?: boolean | null; + trusted?: boolean | null; + }; + backupVersion?: string | null; + crossSigningVerified?: boolean; + encryptionEnabled?: boolean; + pendingVerifications?: number; + recoveryKeyStored?: boolean; + serverDeviceKnown?: boolean; + verified?: boolean; + signedByOwner?: boolean; + deviceId?: string | null; + userId?: string | null; +}; +export type MatrixQaCliEncryptionSetupStatus = { + accountId?: string; + bootstrap?: { + error?: string; + success?: boolean; + }; + configPath?: string; + encryptionChanged?: boolean; + status?: MatrixQaCliVerificationStatus; + success?: boolean; +}; +export type MatrixQaCliAccountAddStatus = { + accountId?: string; + configPath?: string; + encryptionEnabled?: boolean; + verificationBootstrap?: { + attempted?: boolean; + backupVersion?: string | null; + error?: string; + success?: boolean; + }; +}; +export type MatrixQaCliBackupRestoreStatus = { + success?: boolean; + backup?: MatrixQaCliVerificationStatus["backup"]; + error?: string; +}; + +export function isMatrixQaCliBackupUsable( + backup: MatrixQaCliVerificationStatus["backup"], + opts: { allowUntrustedMatchingKey?: boolean } = {}, +): boolean { + return Boolean( + (backup?.trusted || opts.allowUntrustedMatchingKey === true) && + backup?.matchesDecryptionKey && + backup.decryptionKeyCached && + !backup.keyLoadError, + ); +} + +function parseMatrixQaCliJsonText(text: string): unknown { + const candidate = text.trim(); + if (!candidate) { + throw new Error("no JSON payload found"); + } + return JSON.parse(candidate) as unknown; +} + +export function parseMatrixQaCliJson(result: MatrixQaCliRunResult): unknown { + const stdout = result.stdout.trim(); + const stderr = result.stderr.trim(); + if (stdout) { + try { + return parseMatrixQaCliJsonText(stdout); + } catch (error) { + throw new Error( + `${formatMatrixQaCliCommand(result.args)} printed invalid JSON: ${ + error instanceof Error ? error.message : String(error) + }\nstdout:\n${redactMatrixQaCliOutput(stdout)}`, + { cause: error }, + ); + } + } + + if (!stderr) { + throw new Error(`${formatMatrixQaCliCommand(result.args)} did not print JSON`); + } + try { + return parseMatrixQaCliJsonText(stderr); + } catch (error) { + throw new Error( + `${formatMatrixQaCliCommand(result.args)} printed invalid JSON: ${ + error instanceof Error ? error.message : String(error) + }\nstderr:\n${redactMatrixQaCliOutput(stderr)}`, + { cause: error }, + ); + } +} + +export function buildMatrixQaPluginActivationConfig() { + return { + plugins: { + allow: ["matrix"], + entries: { + matrix: { enabled: true }, + }, + }, + }; +} + +export function buildMatrixQaEmptyMatrixCliConfig() { + return { + ...buildMatrixQaPluginActivationConfig(), + channels: { + matrix: { + enabled: true, + accounts: {}, + }, + }, + }; +} + +export async function registerMatrixQaCliE2eeAccount(params: { + context: MatrixQaScenarioContext; + deviceName: string; + scenarioId: MatrixQaE2eeScenarioId; +}) { + const localpartSuffix = params.scenarioId + .replace(/^matrix-e2ee-cli-/, "") + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 24); + const account = await createMatrixQaClient({ + baseUrl: params.context.baseUrl, + }).registerWithToken({ + deviceName: params.deviceName, + localpart: `qa-cli-${localpartSuffix}-${randomUUID().replaceAll("-", "").slice(0, 8)}`, + password: `matrix-qa-${randomUUID()}`, + registrationToken: requireMatrixQaRegistrationToken(params.context), + }); + if (!account.deviceId) { + throw new Error( + `Matrix CLI QA registration for ${params.scenarioId} did not return a device id`, + ); + } + return account; +} + +export async function createMatrixQaE2eeCliOwnerClient(params: { + account: Awaited>; + context: MatrixQaScenarioContext; + scenarioId: MatrixQaE2eeScenarioId; +}) { + return await createMatrixQaE2eeScenarioClient({ + accessToken: params.account.accessToken, + actorId: `cli-owner-${randomUUID().slice(0, 8)}`, + baseUrl: params.context.baseUrl, + deviceId: params.account.deviceId, + observedEvents: params.context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(params.context), + password: params.account.password, + scenarioId: params.scenarioId, + timeoutMs: params.context.timeoutMs, + userId: params.account.userId, + }); +} + +export function parseMatrixQaCliSasText( + text: string, + label: string, +): { kind: "emoji"; value: string } | { kind: "decimal"; value: string } { + const emoji = text.match(/^SAS emoji:\s*(.+)$/m)?.[1]?.trim(); + if (emoji) { + return { kind: "emoji", value: emoji }; + } + const decimal = text.match(/^SAS decimals:\s*(.+)$/m)?.[1]?.trim(); + if (decimal) { + return { kind: "decimal", value: decimal }; + } + throw new Error(`${label} did not print SAS emoji or decimals`); +} + +export function parseMatrixQaCliSummaryField(text: string, field: string): string | null { + const escaped = field.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return text.match(new RegExp(`^${escaped}:\\s*(.+)$`, "m"))?.[1]?.trim() ?? null; +} + +export async function writeMatrixQaCliOutputArtifacts(params: { + label: string; + result: MatrixQaCliRunResult; + rootDir: string; +}) { + await mkdir(params.rootDir, { mode: 0o700, recursive: true }); + await chmod(params.rootDir, 0o700).catch(() => undefined); + const prefix = params.label.replace(/[^A-Za-z0-9_-]/g, "-"); + const stdoutPath = path.join(params.rootDir, `${prefix}.stdout.txt`); + const stderrPath = path.join(params.rootDir, `${prefix}.stderr.txt`); + await Promise.all([ + writeFile(stdoutPath, redactMatrixQaCliOutput(params.result.stdout), { mode: 0o600 }), + writeFile(stderrPath, redactMatrixQaCliOutput(params.result.stderr), { mode: 0o600 }), + ]); + return { stderrPath, stdoutPath }; +} + +export async function assertMatrixQaPrivatePathMode(pathToCheck: string, label: string) { + if (process.platform === "win32") { + return; + } + const mode = (await stat(pathToCheck)).mode & 0o777; + if ((mode & 0o077) !== 0) { + throw new Error(`${label} permissions are too broad: ${mode.toString(8)}`); + } +} + +export function assertMatrixQaCliSasMatches(params: { + cliSas: ReturnType; + owner: MatrixVerificationSummary; +}) { + if (params.cliSas.kind === "emoji") { + const ownerEmoji = formatMatrixQaSasEmoji(params.owner).join(" | "); + if (!ownerEmoji) { + throw new Error("Matrix owner client did not expose SAS emoji"); + } + if (params.cliSas.value !== ownerEmoji) { + throw new Error("Matrix CLI SAS emoji did not match the owner client"); + } + return ownerEmoji.split(" | "); + } + + const ownerDecimal = params.owner.sas?.decimal?.join(" "); + if (!ownerDecimal) { + throw new Error("Matrix owner client did not expose SAS decimals"); + } + if (params.cliSas.value !== ownerDecimal) { + throw new Error("Matrix CLI SAS decimals did not match the owner client"); + } + return [ownerDecimal]; +} + +export function isMatrixQaCliOwnerSelfVerification(params: { + cliDeviceId?: string; + ownerUserId: string; + requireCompleted?: boolean; + requirePending?: boolean; + requireSas?: boolean; + summary: MatrixVerificationSummary; + transactionId?: string; +}) { + const summary = params.summary; + if ( + !summary.isSelfVerification || + summary.initiatedByMe || + summary.otherUserId !== params.ownerUserId + ) { + return false; + } + if (params.transactionId) { + if (summary.transactionId !== params.transactionId) { + return false; + } + } else if (params.cliDeviceId && summary.otherDeviceId !== params.cliDeviceId) { + return false; + } + if (params.requirePending === true && !summary.pending) { + return false; + } + if (params.requireSas === true && !summary.hasSas) { + return false; + } + return params.requireCompleted !== true || summary.completed; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-verification.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-verification.ts new file mode 100644 index 000000000000..dd649f300ff0 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-cli-verification.ts @@ -0,0 +1,269 @@ +// Qa Matrix plugin module implements self-verification CLI E2EE scenarios. +import { createMatrixQaClient } from "../../substrate/client.js"; +import { createMatrixQaCliSelfVerificationRuntime } from "./scenario-runtime-e2ee-cli-runtime.js"; +import { + assertMatrixQaCliSasMatches, + createMatrixQaE2eeCliOwnerClient, + isMatrixQaCliBackupUsable, + isMatrixQaCliOwnerSelfVerification, + parseMatrixQaCliJson, + parseMatrixQaCliSasText, + parseMatrixQaCliSummaryField, + registerMatrixQaCliE2eeAccount, + type MatrixQaCliBackupRestoreStatus, + type MatrixQaCliVerificationStatus, + writeMatrixQaCliOutputArtifacts, +} from "./scenario-runtime-e2ee-cli-shared.js"; +import { + ensureMatrixQaE2eeOwnDeviceVerified, + waitForMatrixQaVerificationSummary, +} from "./scenario-runtime-e2ee-shared.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runMatrixQaE2eeCliSelfVerificationScenario( + context: MatrixQaScenarioContext, +): Promise { + const accountId = "cli"; + const account = await registerMatrixQaCliE2eeAccount({ + context, + deviceName: "OpenClaw Matrix QA CLI Self Verification Owner", + scenarioId: "matrix-e2ee-cli-self-verification", + }); + const owner = await createMatrixQaE2eeCliOwnerClient({ + account, + context, + scenarioId: "matrix-e2ee-cli-self-verification", + }); + try { + const ownerReady = await ensureMatrixQaE2eeOwnDeviceVerified({ + client: owner, + label: "CLI self-verification owner", + }); + const encodedRecoveryKey = ownerReady.recoveryKey?.encodedPrivateKey?.trim(); + if (!encodedRecoveryKey) { + throw new Error("Matrix E2EE self-verification scenario did not expose a recovery key"); + } + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const cliDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA CLI Self Verification Device", + password: account.password, + userId: account.userId, + }); + if (!cliDevice.deviceId) { + throw new Error("Matrix E2EE CLI verification login did not return a device id"); + } + + const cli = await createMatrixQaCliSelfVerificationRuntime({ + accountId, + accessToken: cliDevice.accessToken, + context, + deviceId: cliDevice.deviceId, + userId: cliDevice.userId, + }); + try { + const restoreResult = await cli.run( + [ + "matrix", + "verify", + "backup", + "restore", + "--account", + accountId, + "--recovery-key-stdin", + "--json", + ], + context.timeoutMs, + `${encodedRecoveryKey}\n`, + ); + const restoreArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "verify-backup-restore", + result: restoreResult, + rootDir: cli.rootDir, + }); + const restored = parseMatrixQaCliJson(restoreResult) as MatrixQaCliBackupRestoreStatus; + if ( + restored.success !== true || + restored.backup?.decryptionKeyCached !== true || + restored.backup?.matchesDecryptionKey !== true || + restored.backup?.keyLoadError + ) { + throw new Error( + `Matrix CLI recovery key did not load matching room-key backup material before self-verification: ${ + restored.error ?? restored.backup?.keyLoadError ?? "unknown backup state" + }`, + ); + } + const session = cli.start( + [ + "matrix", + "verify", + "self", + "--account", + accountId, + "--timeout-ms", + String(context.timeoutMs), + ], + context.timeoutMs * 2, + ); + try { + const requestOutput = await session.waitForOutput( + (output) => output.text.includes("Accept this verification request"), + "self-verification request guidance", + context.timeoutMs, + ); + const cliTransactionId = parseMatrixQaCliSummaryField(requestOutput.text, "Transaction id"); + const ownerRequested = await waitForMatrixQaVerificationSummary({ + client: owner, + label: "owner received CLI self-verification request", + predicate: (summary) => + isMatrixQaCliOwnerSelfVerification({ + cliDeviceId: cliTransactionId ? undefined : cliDevice.deviceId, + ownerUserId: account.userId, + requirePending: true, + summary, + transactionId: cliTransactionId ?? undefined, + }), + timeoutMs: context.timeoutMs, + }); + if (ownerRequested.canAccept) { + await owner.acceptVerification(ownerRequested.id); + } + + const sasOutput = await session.waitForOutput( + (output) => /^SAS (?:emoji|decimals):/m.test(output.text), + "SAS emoji or decimals", + context.timeoutMs, + ); + const cliSas = parseMatrixQaCliSasText( + sasOutput.text, + "interactive openclaw matrix verify self", + ); + const ownerSas = await waitForMatrixQaVerificationSummary({ + client: owner, + label: "owner SAS for CLI self-verification", + predicate: (summary) => + isMatrixQaCliOwnerSelfVerification({ + cliDeviceId: cliTransactionId ? undefined : cliDevice.deviceId, + ownerUserId: account.userId, + requireSas: true, + summary, + transactionId: cliTransactionId ?? undefined, + }), + timeoutMs: context.timeoutMs, + }); + const sasArtifact = assertMatrixQaCliSasMatches({ + cliSas, + owner: ownerSas, + }); + const ownerConfirm = owner.confirmVerificationSas(ownerSas.id); + await session.writeStdin("yes\n"); + session.endStdin(); + await ownerConfirm; + const completedCli = await session.wait(); + const selfVerificationArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "verify-self", + result: completedCli, + rootDir: cli.rootDir, + }); + if (!/^Device verified by owner:\s*yes$/m.test(completedCli.stdout)) { + throw new Error( + "Interactive Matrix CLI self-verification did not report final device verification", + ); + } + if (!/^Cross-signing verified:\s*yes$/m.test(completedCli.stdout)) { + throw new Error( + "Interactive Matrix CLI self-verification did not report full Matrix identity trust", + ); + } + const completedOwner = await waitForMatrixQaVerificationSummary({ + client: owner, + label: "owner completed CLI self-verification", + predicate: (summary) => + isMatrixQaCliOwnerSelfVerification({ + cliDeviceId: cliTransactionId ? undefined : cliDevice.deviceId, + ownerUserId: account.userId, + requireCompleted: true, + summary, + transactionId: cliTransactionId ?? undefined, + }), + timeoutMs: context.timeoutMs, + }); + const cliVerificationId = + completedCli.stdout.match(/^Verification id:\s*(\S+)/m)?.[1] ?? "interactive-cli"; + const statusResult = await cli.run([ + "matrix", + "verify", + "status", + "--account", + accountId, + "--json", + ]); + const statusArtifacts = await writeMatrixQaCliOutputArtifacts({ + label: "verify-status", + result: statusResult, + rootDir: cli.rootDir, + }); + const status = parseMatrixQaCliJson(statusResult) as MatrixQaCliVerificationStatus; + if ( + status.verified !== true || + status.crossSigningVerified !== true || + status.signedByOwner !== true || + status.backup?.trusted !== true || + status.backup?.matchesDecryptionKey !== true || + status.backup?.keyLoadError + ) { + throw new Error( + `Matrix CLI device was not fully usable after SAS completion: ownerVerified=${ + status.verified === true && + status.crossSigningVerified === true && + status.signedByOwner === true + ? "yes" + : "no" + }, backupUsable=${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}${ + status.backup?.keyLoadError ? `, backupError=${status.backup.keyLoadError}` : "" + }`, + ); + } + return { + artifacts: { + completedVerificationIds: [cliVerificationId, completedOwner.id], + currentDeviceId: status.deviceId ?? cliDevice.deviceId, + ...(cliSas.kind === "emoji" ? { sasEmoji: sasArtifact } : {}), + secondaryDeviceId: cliDevice.deviceId, + }, + details: [ + "Matrix CLI self-verification established full Matrix identity trust through interactive openclaw matrix verify self", + "cli secret config cleaned after run: yes", + `cli backup restore stdout: ${restoreArtifacts.stdoutPath}`, + `cli backup restore stderr: ${restoreArtifacts.stderrPath}`, + `cli verify self stdout: ${selfVerificationArtifacts.stdoutPath}`, + `cli verify self stderr: ${selfVerificationArtifacts.stderrPath}`, + `cli verify status stdout: ${statusArtifacts.stdoutPath}`, + `cli verify status stderr: ${statusArtifacts.stderrPath}`, + `cli device: ${cliDevice.deviceId}`, + `cli verification id: ${cliVerificationId}`, + `owner-side verification id: ${completedOwner.id}`, + `transaction: ${completedOwner.transactionId ?? ""}`, + `cli verified by owner: ${status.verified ? "yes" : "no"}`, + `cli cross-signing verified: ${status.crossSigningVerified ? "yes" : "no"}`, + `cli backup usable: ${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}`, + ].join("\n"), + }; + } finally { + session.kill(); + } + } finally { + try { + await cli.dispose(); + } finally { + await owner.stop().catch(() => undefined); + await owner.deleteOwnDevices([cliDevice.deviceId]).catch(() => undefined); + } + } + } finally { + await owner.stop().catch(() => undefined); + } +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-messages.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-messages.ts new file mode 100644 index 000000000000..1f74febce1b0 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-messages.ts @@ -0,0 +1,439 @@ +// Qa Matrix plugin module implements message scenario runtime E2EE behavior. +import { randomUUID } from "node:crypto"; +import { startMatrixQaFaultProxy } from "../../substrate/fault-proxy.js"; +import { + buildMatrixQaImageUnderstandingPrompt, + createMatrixQaSplitColorImagePng, + hasMatrixQaExpectedColorReply, + MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, +} from "./scenario-media-fixtures.js"; +import { + patchMatrixQaGatewayMatrixAccount, + readMatrixQaGatewayMatrixAccount, + replaceMatrixQaGatewayMatrixAccount, +} from "./scenario-runtime-config.js"; +import { + buildMatrixE2eeReplyArtifact, + buildSyncStateAfterMissingEncryptionFaultRule, + runMatrixQaE2eeTopLevelScenario, + runMatrixQaE2eeTopLevelWithClient, + withMatrixQaE2eeDriver, + withMatrixQaIsolatedE2eeDriverRoom, +} from "./scenario-runtime-e2ee-room.js"; +import { + MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID, + MATRIX_QA_SYNC_STATE_AFTER_PARAM, + isMatrixQaE2eeNoticeTriggeredSutReply, + requireMatrixQaGatewayConfigPath, + resolveMatrixQaE2eeScenarioGroupRoom, +} from "./scenario-runtime-e2ee-shared.js"; +import { + assertThreadReplyArtifact, + buildMatrixQaToken, + buildMatrixReplyDetails, + buildMentionPrompt, + isMatrixQaExactMarkerReply, + resolveMatrixQaNoReplyWindowMs, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaReplyArtifact, MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runMatrixQaE2eeBasicReplyScenario( + context: MatrixQaScenarioContext, +): Promise { + const result = await runMatrixQaE2eeTopLevelScenario(context, { + scenarioId: "matrix-e2ee-basic-reply", + tokenPrefix: "MATRIX_QA_E2EE_BASIC", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: result.roomKey, + roomId: result.roomId, + }, + details: [ + `encrypted room key: ${result.roomKey}`, + `encrypted room id: ${result.roomId}`, + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("E2EE reply", result.reply), + ].join("\n"), + }; +} + +export async function runMatrixQaE2eeStateAfterMissingEncryptionScenario( + context: MatrixQaScenarioContext, +): Promise { + if (!context.restartGatewayAfterStateMutation) { + throw new Error("Matrix E2EE state_after QA scenario requires hard gateway restart support"); + } + const accountId = context.sutAccountId ?? "sut"; + const configPath = requireMatrixQaGatewayConfigPath(context); + const originalAccountConfig = await readMatrixQaGatewayMatrixAccount({ + accountId, + configPath, + }); + const proxy = await startMatrixQaFaultProxy({ + targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl, + ...context.faultProxyObserver, + rules: [buildSyncStateAfterMissingEncryptionFaultRule(context.sutAccessToken)], + }); + let gatewayPatched = false; + try { + await context.restartGatewayAfterStateMutation( + async () => { + await patchMatrixQaGatewayMatrixAccount({ + accountId, + accountPatch: { + homeserver: proxy.baseUrl, + network: { + dangerouslyAllowPrivateNetwork: true, + }, + }, + configPath, + }); + gatewayPatched = true; + }, + { + timeoutMs: context.timeoutMs, + waitAccountId: accountId, + }, + ); + const result = await runMatrixQaE2eeTopLevelScenario(context, { + scenarioId: "matrix-e2ee-state-after-missing-encryption", + tokenPrefix: "MATRIX_QA_E2EE_STATE_AFTER", + }); + const stateAfterHits = proxy + .hits() + .filter((hit) => hit.ruleId === MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID); + if (stateAfterHits.length > 0) { + throw new Error( + `Matrix E2EE gateway still sent ${MATRIX_QA_SYNC_STATE_AFTER_PARAM}=true on /sync`, + ); + } + return { + artifacts: { + driverEventId: result.driverEventId, + faultProxyBaseUrl: proxy.baseUrl, + reply: result.reply, + roomKey: result.roomKey, + roomId: result.roomId, + stateAfterFaultHitCount: stateAfterHits.length, + stateAfterFaultRuleId: MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID, + strippedSyncStateAfterParam: true, + }, + details: [ + `encrypted room key: ${result.roomKey}`, + `encrypted room id: ${result.roomId}`, + `driver event: ${result.driverEventId}`, + `fault proxy: ${proxy.baseUrl}`, + `state_after sync opt-in hits: ${stateAfterHits.length}`, + ...buildMatrixReplyDetails("E2EE state_after reply", result.reply), + ].join("\n"), + }; + } finally { + if (gatewayPatched) { + await context + .restartGatewayAfterStateMutation( + async () => { + await replaceMatrixQaGatewayMatrixAccount({ + accountConfig: originalAccountConfig, + accountId, + configPath, + }); + }, + { + timeoutMs: context.timeoutMs, + waitAccountId: accountId, + }, + ) + .catch(() => undefined); + } + await proxy.stop().catch(() => undefined); + } +} + +export async function runMatrixQaE2eeThreadFollowUpScenario( + context: MatrixQaScenarioContext, +): Promise { + const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-thread-follow-up", + ); + const result = await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-thread-follow-up", + async (client) => { + await client.prime(); + const rootEventId = await client.sendTextMessage({ + body: `E2EE thread root ${randomUUID().slice(0, 8)}`, + roomId, + }); + const token = buildMatrixQaToken("MATRIX_QA_E2EE_THREAD"); + const driverEventId = await client.sendTextMessage({ + body: buildMentionPrompt(context.sutUserId, token), + mentionUserIds: [context.sutUserId], + replyToEventId: rootEventId, + roomId, + threadRootEventId: rootEventId, + }); + const matched = await client.waitForRoomEvent({ + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId, + sutUserId: context.sutUserId, + token, + }) && + event.relatesTo?.relType === "m.thread" && + event.relatesTo.eventId === rootEventId, + roomId, + timeoutMs: context.timeoutMs, + }); + const reply = buildMatrixE2eeReplyArtifact(matched.event, token); + assertThreadReplyArtifact(reply, { + expectedRootEventId: rootEventId, + label: "E2EE threaded reply", + }); + return { + driverEventId, + reply, + rootEventId, + token, + }; + }, + ); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + rootEventId: result.rootEventId, + roomKey, + roomId, + }, + details: [ + `encrypted room key: ${roomKey}`, + `encrypted room id: ${roomId}`, + `thread root event: ${result.rootEventId}`, + `mention trigger event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("E2EE threaded reply", result.reply), + ].join("\n"), + }; +} + +export async function runMatrixQaE2eeRestartResumeScenario( + context: MatrixQaScenarioContext, +): Promise { + if (!context.restartGateway) { + throw new Error("Matrix E2EE restart scenario requires gateway restart support"); + } + const restartGateway = context.restartGateway; + return await withMatrixQaIsolatedE2eeDriverRoom( + context, + "matrix-e2ee-restart-resume", + async ({ client, driverUserId, roomId, roomKey }) => { + const first = await runMatrixQaE2eeTopLevelWithClient(context, { + client, + driverUserId, + roomId, + roomKey, + tokenPrefix: "MATRIX_QA_E2EE_BEFORE_RESTART", + }); + await restartGateway(); + const recovered = await runMatrixQaE2eeTopLevelWithClient(context, { + client, + driverUserId, + roomId, + roomKey, + tokenPrefix: "MATRIX_QA_E2EE_AFTER_RESTART", + }); + return { + artifacts: { + driverUserId, + firstDriverEventId: first.driverEventId, + firstReply: first.reply, + recoveredDriverEventId: recovered.driverEventId, + recoveredReply: recovered.reply, + restartSignal: "gateway-restart", + roomKey: recovered.roomKey, + roomId: recovered.roomId, + }, + details: [ + `encrypted room key: ${recovered.roomKey}`, + `encrypted room id: ${recovered.roomId}`, + `isolated driver user: ${driverUserId}`, + `pre-restart event: ${first.driverEventId}`, + ...buildMatrixReplyDetails("pre-restart reply", first.reply), + `post-restart event: ${recovered.driverEventId}`, + ...buildMatrixReplyDetails("post-restart reply", recovered.reply), + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeVerificationNoticeNoTriggerScenario( + context: MatrixQaScenarioContext, +): Promise { + const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-verification-notice-no-trigger", + ); + return await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-verification-notice-no-trigger", + async (client) => { + await client.prime(); + const token = buildMatrixQaToken("MATRIX_QA_E2EE_VERIFY_NOTICE"); + const body = `Matrix verification started with ${context.driverUserId}; ${buildMentionPrompt( + context.sutUserId, + token, + )}`; + const noticeSentAt = Date.now(); + const noticeEventId = await client.sendNoticeMessage({ + body, + mentionUserIds: [context.sutUserId], + roomId, + }); + const result = await client.waitForOptionalRoomEvent({ + predicate: (event) => + isMatrixQaE2eeNoticeTriggeredSutReply({ + event, + noticeEventId, + noticeSentAt, + roomId, + sutUserId: context.sutUserId, + token, + }), + roomId, + timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs), + }); + if (result.matched) { + throw new Error(`unexpected E2EE verification-notice reply: ${result.event.eventId}`); + } + return { + artifacts: { + expectedNoReplyWindowMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs), + noticeEventId, + roomKey, + roomId, + }, + details: [ + `encrypted room key: ${roomKey}`, + `encrypted room id: ${roomId}`, + `verification notice event: ${noticeEventId}`, + `waited ${resolveMatrixQaNoReplyWindowMs(context.timeoutMs)}ms with no SUT reply`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeArtifactRedactionScenario( + context: MatrixQaScenarioContext, +): Promise { + return await withMatrixQaIsolatedE2eeDriverRoom( + context, + "matrix-e2ee-artifact-redaction", + async ({ client, driverUserId, roomId, roomKey }) => { + const result = await runMatrixQaE2eeTopLevelWithClient(context, { + client, + driverUserId, + roomId, + roomKey, + tokenPrefix: "MATRIX_QA_E2EE_REDACT", + }); + const leaked = context.observedEvents.some( + (event) => + event.roomId === result.roomId && + (event.body?.includes(result.token) || event.formattedBody?.includes(result.token)), + ); + if (!leaked) { + throw new Error( + "Matrix E2EE redaction scenario did not observe decrypted content in memory", + ); + } + return { + artifacts: { + driverEventId: result.driverEventId, + driverUserId, + reply: result.reply, + roomKey: result.roomKey, + roomId: result.roomId, + }, + details: [ + "decrypted E2EE payload reached in-memory assertions only", + "observed-event artifacts redact body/formatted_body unless OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1", + `encrypted room id: ${result.roomId}`, + `isolated driver user: ${driverUserId}`, + ...buildMatrixReplyDetails("E2EE reply", result.reply), + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeMediaImageScenario( + context: MatrixQaScenarioContext, +): Promise { + return await withMatrixQaIsolatedE2eeDriverRoom( + context, + "matrix-e2ee-media-image", + async ({ client, driverUserId, roomId, roomKey }) => { + const startSince = await client.prime(); + const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId); + const driverEventId = await client.sendImageMessage({ + body: triggerBody, + buffer: createMatrixQaSplitColorImagePng(), + contentType: "image/png", + fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + mentionUserIds: [context.sutUserId], + roomId, + }); + const attachmentEvent = await client.waitForRoomEvent({ + predicate: (event) => + event.roomId === roomId && + event.eventId === driverEventId && + event.sender === driverUserId && + event.attachment?.kind === "image" && + event.attachment.caption === triggerBody, + roomId, + timeoutMs: context.timeoutMs, + }); + const matched = await client.waitForRoomEvent({ + predicate: (event) => + event.roomId === roomId && + event.sender === context.sutUserId && + event.type === "m.room.message" && + event.relatesTo === undefined && + hasMatrixQaExpectedColorReply(event.body), + roomId, + timeoutMs: context.timeoutMs, + }); + const reply: MatrixQaReplyArtifact = { + eventId: matched.event.eventId, + mentions: matched.event.mentions, + relatesTo: matched.event.relatesTo, + sender: matched.event.sender, + }; + return { + artifacts: { + attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + driverEventId, + driverUserId, + reply, + roomKey, + roomId, + }, + details: [ + `encrypted room key: ${roomKey}`, + `encrypted room id: ${roomId}`, + `isolated driver user: ${driverUserId}`, + `driver encrypted image event: ${driverEventId}`, + `driver encrypted image filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`, + `driver encrypted image since: ${attachmentEvent.since ?? startSince ?? ""}`, + ...buildMatrixReplyDetails("E2EE image reply", reply), + ].join("\n"), + }; + }, + ); +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-recovery.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-recovery.ts new file mode 100644 index 000000000000..57b584ed9777 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-recovery.ts @@ -0,0 +1,285 @@ +// Qa Matrix plugin module implements recovery scenario runtime E2EE behavior. +import { randomUUID } from "node:crypto"; +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { createMatrixQaE2eeScenarioClient } from "../../substrate/e2ee-client.js"; +import { + assertMatrixQaExpectedBootstrapFailure, + assertMatrixQaFaultedRecoveryOwnerVerificationRequired, + runMatrixQaFaultedE2eeBootstrap, + runMatrixQaFaultedRecoveryOwnerVerification, + withMatrixQaE2eeDriver, +} from "./scenario-runtime-e2ee-room.js"; +import { + assertMatrixQaBootstrapSucceeded, + ensureMatrixQaE2eeOwnDeviceVerified, + MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID, + MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT, + requireMatrixQaE2eeOutputDir, + requireMatrixQaPassword, + resolveMatrixQaE2eeScenarioGroupRoom, + waitForMatrixQaNonEmptyRoomKeyRestore, +} from "./scenario-runtime-e2ee-shared.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runMatrixQaE2eeBootstrapSuccessScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + return await withMatrixQaE2eeDriver(context, "matrix-e2ee-bootstrap-success", async (client) => { + const initial = await client.bootstrapOwnDeviceVerification(); + assertMatrixQaBootstrapSucceeded("driver initial", initial); + const result = await client.bootstrapOwnDeviceVerification({ + forceResetCrossSigning: true, + }); + assertMatrixQaBootstrapSucceeded("driver", result); + return { + artifacts: { + backupCreatedVersion: result.verification.backupVersion, + bootstrapActor: "driver", + bootstrapSuccess: true, + currentDeviceId: result.verification.deviceId, + recoveryKeyId: result.verification.recoveryKeyId, + recoveryKeyStored: result.verification.recoveryKeyStored, + }, + details: [ + "driver bootstrap and guarded cross-signing reset succeeded through real Matrix crypto bootstrap", + `device verified: ${result.verification.verified ? "yes" : "no"}`, + `cross-signing verified: ${result.verification.crossSigningVerified ? "yes" : "no"}`, + `signed by owner: ${result.verification.signedByOwner ? "yes" : "no"}`, + `cross-signing published: ${result.crossSigning.published ? "yes" : "no"}`, + `room-key backup version: ${result.verification.backupVersion ?? ""}`, + `recovery key id: ${result.verification.recoveryKeyId ?? ""}`, + ].join("\n"), + }; + }); +} + +export async function runMatrixQaE2eeRecoveryKeyLifecycleScenario( + context: MatrixQaScenarioContext, +): Promise { + const driverPassword = requireMatrixQaPassword(context, "driver"); + return await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-recovery-key-lifecycle", + async (client) => { + const { roomId } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-recovery-key-lifecycle", + ); + const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ + client, + label: "driver", + }); + const recoveryKey = ready.recoveryKey; + const encodedRecoveryKey = recoveryKey?.encodedPrivateKey?.trim(); + if (!encodedRecoveryKey) { + throw new Error("Matrix E2EE bootstrap did not expose an encoded recovery key"); + } + const seededEventId = await client.sendTextMessage({ + body: `E2EE recovery-key restore seed ${randomUUID().slice(0, 8)}`, + roomId, + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const recoveryDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA Recovery Restore Device", + password: driverPassword, + userId: context.driverUserId, + }); + if (!recoveryDevice.deviceId) { + throw new Error("Matrix E2EE recovery login did not return a secondary device id"); + } + const recoveryClient = await createMatrixQaE2eeScenarioClient({ + accessToken: recoveryDevice.accessToken, + actorId: `driver-recovery-${randomUUID().slice(0, 8)}`, + baseUrl: context.baseUrl, + deviceId: recoveryDevice.deviceId, + observedEvents: context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(context), + password: recoveryDevice.password, + scenarioId: "matrix-e2ee-recovery-key-lifecycle", + timeoutMs: context.timeoutMs, + userId: recoveryDevice.userId, + }); + let cleanupRecoveryDevice = true; + try { + const recoveryVerification = await recoveryClient.verifyWithRecoveryKey(encodedRecoveryKey); + if (!recoveryVerification.success) { + throw new Error( + `Matrix E2EE recovery device verification failed: ${recoveryVerification.error ?? "unknown error"}`, + ); + } + const restored = await waitForMatrixQaNonEmptyRoomKeyRestore({ + client: recoveryClient, + recoveryKey: encodedRecoveryKey, + timeoutMs: context.timeoutMs, + }); + const reset = await recoveryClient.resetRoomKeyBackup(); + if (!reset.success) { + throw new Error( + `Matrix E2EE room-key backup reset failed: ${reset.error ?? "unknown error"}`, + ); + } + const resetRecoveryKey = await recoveryClient.getRecoveryKey(); + const resetEncodedRecoveryKey = resetRecoveryKey?.encodedPrivateKey?.trim(); + if (resetEncodedRecoveryKey && resetEncodedRecoveryKey !== encodedRecoveryKey) { + const ownerRecovery = await client.verifyWithRecoveryKey(resetEncodedRecoveryKey); + if (!ownerRecovery.success) { + throw new Error( + `Matrix E2EE owner could not refresh recovery key after backup reset: ${ + ownerRecovery.error ?? "unknown error" + }`, + ); + } + } + await recoveryClient.stop(); + await client.stop().catch(() => undefined); + await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); + cleanupRecoveryDevice = false; + return { + artifacts: { + backupCreatedVersion: reset.createdVersion, + backupReset: reset.success, + backupRestored: restored.success, + bootstrapActor: "driver", + bootstrapSuccess: ready.bootstrap?.success ?? true, + recoveryDeviceId: recoveryDevice.deviceId, + recoveryKeyId: recoveryKey?.keyId ?? null, + recoveryKeyUsable: + recoveryVerification.recoveryKeyAccepted && recoveryVerification.backupUsable, + recoveryKeyStored: true, + recoveryVerified: recoveryVerification.deviceOwnerVerified, + restoreImported: restored.imported, + restoreTotal: restored.total, + seededEventId, + }, + details: [ + "driver recovery lifecycle completed through real Matrix recovery APIs", + `bootstrap backup version: ${ready.verification.backupVersion ?? ""}`, + `seeded encrypted event: ${seededEventId}`, + `recovery device: ${recoveryDevice.deviceId}`, + `recovery key usable: ${recoveryVerification.backupUsable ? "yes" : "no"}`, + `recovery device verified: ${recoveryVerification.deviceOwnerVerified ? "yes" : "no"}`, + `restore imported/total: ${restored.imported}/${restored.total}`, + `restore loaded from secret storage: ${restored.loadedFromSecretStorage ? "yes" : "no"}`, + `reset previous version: ${reset.previousVersion ?? ""}`, + `reset created version: ${reset.createdVersion ?? ""}`, + ].join("\n"), + }; + } finally { + if (cleanupRecoveryDevice) { + await recoveryClient.stop().catch(() => undefined); + await client.stop().catch(() => undefined); + await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); + } + } + }, + ); +} + +export async function runMatrixQaE2eeRecoveryOwnerVerificationRequiredScenario( + context: MatrixQaScenarioContext, +): Promise { + const driverPassword = requireMatrixQaPassword(context, "driver"); + return await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-recovery-owner-verification-required", + async (client) => { + const { roomId } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-recovery-owner-verification-required", + ); + const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ + client, + label: "driver", + }); + const recoveryKey = ready.recoveryKey; + const encodedRecoveryKey = recoveryKey?.encodedPrivateKey?.trim(); + if (!encodedRecoveryKey) { + throw new Error("Matrix E2EE bootstrap did not expose an encoded recovery key"); + } + const seededEventId = await client.sendTextMessage({ + body: `E2EE recovery owner-verification seed ${randomUUID().slice(0, 8)}`, + roomId, + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const recoveryDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA Owner Verification Required Device", + password: driverPassword, + userId: context.driverUserId, + }); + if (!recoveryDevice.deviceId) { + throw new Error("Matrix E2EE recovery login did not return a secondary device id"); + } + try { + const faulted = await runMatrixQaFaultedRecoveryOwnerVerification({ + accessToken: recoveryDevice.accessToken, + context, + deviceId: recoveryDevice.deviceId, + encodedRecoveryKey, + userId: recoveryDevice.userId, + }); + assertMatrixQaFaultedRecoveryOwnerVerificationRequired(faulted); + return { + artifacts: { + backupRestored: faulted.restore.success, + backupUsable: faulted.verification.backupUsable, + faultHitCount: faulted.faultHits.length, + faultedEndpoints: faulted.faultHits.map((hit) => hit.path), + faultRuleId: MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID, + recoveryDeviceId: recoveryDevice.deviceId, + recoveryKeyAccepted: faulted.verification.recoveryKeyAccepted, + recoveryKeyId: recoveryKey?.keyId ?? null, + recoveryVerified: faulted.verification.deviceOwnerVerified, + restoreImported: faulted.restore.imported, + restoreTotal: faulted.restore.total, + verificationSuccess: faulted.verification.success, + }, + details: [ + "driver recovery key unlocked backup while owner signature upload was blocked", + `seeded encrypted event: ${seededEventId}`, + `recovery device: ${recoveryDevice.deviceId}`, + `fault hits: ${faulted.faultHits.length}`, + `recovery key accepted: ${faulted.verification.recoveryKeyAccepted ? "yes" : "no"}`, + `backup usable: ${faulted.verification.backupUsable ? "yes" : "no"}`, + `device owner verified: ${faulted.verification.deviceOwnerVerified ? "yes" : "no"}`, + `restore imported/total: ${faulted.restore.imported}/${faulted.restore.total}`, + ].join("\n"), + }; + } finally { + await client.stop().catch(() => undefined); + await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); + } + }, + ); +} + +export async function runMatrixQaE2eeKeyBootstrapFailureScenario( + context: MatrixQaScenarioContext, +): Promise { + const { faultHits, result } = await runMatrixQaFaultedE2eeBootstrap(context); + const bootstrapError = assertMatrixQaExpectedBootstrapFailure({ faultHits, result }); + + return { + artifacts: { + bootstrapActor: "driver", + bootstrapErrorPreview: truncateUtf16Safe(bootstrapError, 240), + bootstrapSuccess: result.success, + faultedEndpoint: MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT, + faultHitCount: faultHits.length, + ...(faultHits[0]?.ruleId ? { faultRuleId: faultHits[0].ruleId } : {}), + }, + details: [ + "Matrix E2EE bootstrap failure surfaced through real SDK bootstrap.", + `faulted endpoint: GET ${MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT}`, + `fault hits: ${faultHits.length}`, + `bootstrap success: ${result.success ? "yes" : "no"}`, + `bootstrap error: ${bootstrapError || ""}`, + ].join("\n"), + }; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-room.ts new file mode 100644 index 000000000000..6135dec4784c --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-room.ts @@ -0,0 +1,488 @@ +// Qa Matrix plugin module implements room and fault scenario runtime E2EE behavior. +import { randomUUID } from "node:crypto"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { + createMatrixQaE2eeScenarioClient, + runMatrixQaE2eeBootstrap, +} from "../../substrate/e2ee-client.js"; +import type { MatrixQaE2eeScenarioClient } from "../../substrate/e2ee-client.js"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { + startMatrixQaFaultProxy, + type MatrixQaFaultProxyHit, + type MatrixQaFaultProxyRule, +} from "../../substrate/fault-proxy.js"; +import { + buildMatrixQaE2eeScenarioRoomKey, + type MatrixQaE2eeScenarioId, +} from "./scenario-catalog.js"; +import { + isMatrixQaPlainRecord, + patchMatrixQaGatewayMatrixAccount, + readMatrixQaGatewayMatrixAccount, +} from "./scenario-runtime-config.js"; +import { + MATRIX_QA_KEYS_SIGNATURES_UPLOAD_ENDPOINT, + MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID, + MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID, + MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT, + MATRIX_QA_SYNC_ENDPOINT, + MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID, + MATRIX_QA_SYNC_STATE_AFTER_KEY, + MATRIX_QA_SYNC_STATE_AFTER_PARAM, + createMatrixQaE2eeDriverClient, + requireMatrixQaE2eeOutputDir, + requireMatrixQaGatewayConfigPath, + registerMatrixQaE2eeScenarioAccount, + resolveMatrixQaE2eeScenarioGroupRoom, + waitForMatrixQaNonEmptyRoomKeyRestore, +} from "./scenario-runtime-e2ee-shared.js"; +import { + assertTopLevelReplyArtifact, + buildMatrixQaToken, + buildMentionPrompt, + doesMatrixQaReplyBodyMatchToken, + isMatrixQaExactMarkerReply, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaReplyArtifact } from "./scenario-types.js"; + +type MatrixQaE2eeBootstrapResult = Awaited>; + +export function buildMatrixE2eeReplyArtifact( + event: MatrixQaObservedEvent, + token: string, +): MatrixQaReplyArtifact { + return { + eventId: event.eventId, + mentions: event.mentions, + relatesTo: event.relatesTo, + sender: event.sender, + tokenMatched: doesMatrixQaReplyBodyMatchToken(event, token), + }; +} + +export function buildRoomKeyBackupUnavailableFaultRule( + accessToken: string, +): MatrixQaFaultProxyRule { + return { + id: MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID, + match: (request) => + request.method === "GET" && + request.path === MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT && + request.bearerToken === accessToken, + response: () => ({ + body: { + errcode: "M_NOT_FOUND", + error: "No current key backup", + }, + status: 404, + }), + }; +} + +function buildOwnerSignatureUploadBlockedFaultRule(accessToken: string): MatrixQaFaultProxyRule { + return { + id: MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID, + match: (request) => + request.method === "POST" && + request.path === MATRIX_QA_KEYS_SIGNATURES_UPLOAD_ENDPOINT && + request.bearerToken === accessToken, + response: () => ({ + body: {}, + status: 200, + }), + }; +} + +function removeMatrixQaSyncStateAfterEncryptionEvents(payload: unknown) { + if (!isMatrixQaPlainRecord(payload)) { + return 0; + } + const rooms = isMatrixQaPlainRecord(payload.rooms) ? payload.rooms : {}; + const join = isMatrixQaPlainRecord(rooms.join) ? rooms.join : {}; + let removed = 0; + for (const room of Object.values(join)) { + if (!isMatrixQaPlainRecord(room)) { + continue; + } + const stateAfter = room[MATRIX_QA_SYNC_STATE_AFTER_KEY]; + if (!isMatrixQaPlainRecord(stateAfter) || !Array.isArray(stateAfter.events)) { + continue; + } + const filtered = stateAfter.events.filter((event) => { + if (isMatrixQaPlainRecord(event) && event.type === "m.room.encryption") { + removed += 1; + return false; + } + return true; + }); + stateAfter.events = filtered; + } + return removed; +} + +export function buildSyncStateAfterMissingEncryptionFaultRule( + accessToken: string, +): MatrixQaFaultProxyRule { + return { + id: MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID, + match: (request) => + request.method === "GET" && + request.path === MATRIX_QA_SYNC_ENDPOINT && + request.bearerToken === accessToken && + new URLSearchParams(request.search).get(MATRIX_QA_SYNC_STATE_AFTER_PARAM) === "true", + mutateResponse: ({ response }) => { + const contentType = response.headers.get("content-type") ?? ""; + if (!contentType.includes("json")) { + return response; + } + const payload = JSON.parse(response.body.toString("utf8")) as unknown; + removeMatrixQaSyncStateAfterEncryptionEvents(payload); + return { + ...response, + body: Buffer.from(JSON.stringify(payload)), + }; + }, + }; +} + +export async function runMatrixQaFaultedE2eeBootstrap(context: MatrixQaScenarioContext): Promise<{ + faultHits: MatrixQaFaultProxyHit[]; + result: MatrixQaE2eeBootstrapResult; +}> { + const proxy = await startMatrixQaFaultProxy({ + targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl, + ...context.faultProxyObserver, + rules: [buildRoomKeyBackupUnavailableFaultRule(context.driverAccessToken)], + }); + try { + const result = await runMatrixQaE2eeBootstrap({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: proxy.baseUrl, + deviceId: context.driverDeviceId, + outputDir: requireMatrixQaE2eeOutputDir(context), + ...(context.driverPassword ? { password: context.driverPassword } : {}), + scenarioId: "matrix-e2ee-key-bootstrap-failure", + timeoutMs: context.timeoutMs, + userId: context.driverUserId, + }); + return { + faultHits: proxy.hits(), + result, + }; + } finally { + await proxy.stop(); + } +} + +export async function runMatrixQaFaultedRecoveryOwnerVerification(params: { + accessToken: string; + context: MatrixQaScenarioContext; + deviceId: string; + encodedRecoveryKey: string; + userId: string; +}): Promise<{ + faultHits: MatrixQaFaultProxyHit[]; + restore: Awaited>; + verification: Awaited>; +}> { + const proxy = await startMatrixQaFaultProxy({ + targetBaseUrl: params.context.faultProxyTargetBaseUrl ?? params.context.baseUrl, + ...params.context.faultProxyObserver, + rules: [buildOwnerSignatureUploadBlockedFaultRule(params.accessToken)], + }); + const recoveryClient = await createMatrixQaE2eeScenarioClient({ + accessToken: params.accessToken, + actorId: `driver-recovery-${randomUUID().slice(0, 8)}`, + baseUrl: proxy.baseUrl, + deviceId: params.deviceId, + observedEvents: params.context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(params.context), + scenarioId: "matrix-e2ee-recovery-owner-verification-required", + timeoutMs: params.context.timeoutMs, + userId: params.userId, + }); + try { + const verification = await recoveryClient.verifyWithRecoveryKey(params.encodedRecoveryKey); + const restore = await waitForMatrixQaNonEmptyRoomKeyRestore({ + client: recoveryClient, + recoveryKey: params.encodedRecoveryKey, + timeoutMs: params.context.timeoutMs, + }); + return { + faultHits: proxy.hits(), + restore, + verification, + }; + } finally { + await recoveryClient.stop().catch(() => undefined); + await proxy.stop(); + } +} + +export function assertMatrixQaFaultedRecoveryOwnerVerificationRequired( + faulted: Awaited>, +) { + if (faulted.faultHits.length === 0) { + throw new Error("Matrix E2EE owner signature fault proxy was not exercised"); + } + if (faulted.verification.success) { + throw new Error( + "Matrix E2EE recovery verification unexpectedly succeeded while owner signature upload was blocked", + ); + } + if (!faulted.verification.recoveryKeyAccepted) { + throw new Error("Matrix E2EE recovery key was not accepted"); + } + if (!faulted.verification.backupUsable) { + throw new Error("Matrix E2EE recovery key did not leave room-key backup usable"); + } + if (faulted.verification.deviceOwnerVerified) { + throw new Error("Matrix E2EE recovery device should still require Matrix identity trust"); + } + if (!faulted.restore.success) { + throw new Error( + `Matrix E2EE room-key backup restore failed after owner-verification fault: ${faulted.restore.error ?? "unknown error"}`, + ); + } +} + +export function assertMatrixQaExpectedBootstrapFailure(params: { + faultHits: MatrixQaFaultProxyHit[]; + result: MatrixQaE2eeBootstrapResult; +}) { + if (params.faultHits.length === 0) { + throw new Error("Matrix E2EE bootstrap fault proxy was not exercised"); + } + if (params.result.success) { + throw new Error( + "Matrix E2EE bootstrap unexpectedly succeeded while room-key backup was faulted", + ); + } + const bootstrapError = params.result.error ?? ""; + if (!bootstrapError.toLowerCase().includes("room key backup")) { + throw new Error(`Matrix E2EE bootstrap failed for an unexpected reason: ${bootstrapError}`); + } + return bootstrapError; +} + +export async function withMatrixQaE2eeDriver( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, + run: (client: MatrixQaE2eeScenarioClient) => Promise, + opts: { actorId?: "driver" | `driver-${string}` } = {}, +) { + const client = await createMatrixQaE2eeDriverClient(context, scenarioId, opts); + try { + return await run(client); + } finally { + await client.stop(); + } +} + +async function createMatrixQaE2eeRegisteredScenarioClient(params: { + account: Awaited>; + actorId: `driver-${string}`; + context: MatrixQaScenarioContext; + scenarioId: MatrixQaE2eeScenarioId; +}) { + return await createMatrixQaE2eeScenarioClient({ + accessToken: params.account.accessToken, + actorId: params.actorId, + baseUrl: params.context.baseUrl, + deviceId: params.account.deviceId, + observedEvents: params.context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(params.context), + password: params.account.password, + scenarioId: params.scenarioId, + timeoutMs: params.context.timeoutMs, + userId: params.account.userId, + }); +} + +export async function withMatrixQaIsolatedE2eeDriverRoom( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, + run: (params: { + client: MatrixQaE2eeScenarioClient; + driverUserId: string; + roomId: string; + roomKey: string; + }) => Promise, +) { + if (!context.restartGatewayAfterStateMutation) { + throw new Error( + "Matrix E2EE isolated driver room scenario requires hard gateway restart support", + ); + } + const accountId = context.sutAccountId ?? "sut"; + const configPath = requireMatrixQaGatewayConfigPath(context); + const accountConfig = await readMatrixQaGatewayMatrixAccount({ + accountId, + configPath, + }); + const originalGroups = isMatrixQaPlainRecord(accountConfig.groups) ? accountConfig.groups : {}; + const originalGroupAllowFrom = Array.isArray(accountConfig.groupAllowFrom) + ? accountConfig.groupAllowFrom + : undefined; + const originalGroupPolicy = accountConfig.groupPolicy; + const driverAccount = await registerMatrixQaE2eeScenarioAccount({ + context, + deviceName: "OpenClaw Matrix QA Isolated E2EE Driver", + localpartPrefix: "qa-e2ee-driver", + scenarioId, + }); + const driverApi = createMatrixQaClient({ + accessToken: driverAccount.accessToken, + baseUrl: context.baseUrl, + }); + const roomKey = buildMatrixQaE2eeScenarioRoomKey(scenarioId); + const roomId = await driverApi.createPrivateRoom({ + encrypted: true, + inviteUserIds: [context.observerUserId, context.sutUserId], + name: `Matrix QA ${scenarioId} Isolated E2EE Room`, + }); + await Promise.all([ + createMatrixQaClient({ + accessToken: context.observerAccessToken, + baseUrl: context.baseUrl, + }).joinRoom(roomId), + createMatrixQaClient({ + accessToken: context.sutAccessToken, + baseUrl: context.baseUrl, + }).joinRoom(roomId), + ]); + + const isolatedGroups = { + [roomId]: { + enabled: true, + requireMention: true, + }, + }; + const applyPatch = async (accountPatch: Record) => { + await context.restartGatewayAfterStateMutation?.( + async () => { + await patchMatrixQaGatewayMatrixAccount({ + accountId, + accountPatch, + configPath, + }); + }, + { + timeoutMs: context.timeoutMs, + waitAccountId: accountId, + }, + ); + }; + + let patchedGateway; + let client: MatrixQaE2eeScenarioClient | undefined; + try { + await applyPatch({ + groupAllowFrom: [driverAccount.userId], + groupPolicy: "allowlist", + groups: isolatedGroups, + }); + patchedGateway = true; + const actorId: `driver-${string}` = `driver-${scenarioId + .replace(/^matrix-e2ee-/, "") + .replace(/[^A-Za-z0-9_-]/g, "-") + .slice(0, 28)}`; + client = await createMatrixQaE2eeRegisteredScenarioClient({ + account: driverAccount, + actorId, + context, + scenarioId, + }); + await Promise.all([ + client.waitForJoinedMember({ + roomId, + timeoutMs: context.timeoutMs, + userId: context.sutUserId, + }), + client.waitForJoinedMember({ + roomId, + timeoutMs: context.timeoutMs, + userId: context.observerUserId, + }), + ]); + return await run({ + client, + driverUserId: driverAccount.userId, + roomId, + roomKey, + }); + } finally { + await client?.stop().catch(() => undefined); + if (patchedGateway) { + const restorePatch: Record = { + groupAllowFrom: originalGroupAllowFrom, + groupPolicy: originalGroupPolicy, + groups: originalGroups, + }; + await applyPatch(restorePatch).catch(() => undefined); + } + } +} + +export async function runMatrixQaE2eeTopLevelWithClient( + context: MatrixQaScenarioContext, + params: { + client: MatrixQaE2eeScenarioClient; + driverUserId: string; + roomId: string; + roomKey: string; + tokenPrefix: string; + }, +) { + const startSince = await params.client.prime(); + const token = buildMatrixQaToken(params.tokenPrefix); + const body = buildMentionPrompt(context.sutUserId, token); + const driverEventId = await params.client.sendTextMessage({ + body, + mentionUserIds: [context.sutUserId], + roomId: params.roomId, + }); + const matched = await params.client.waitForRoomEvent({ + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId: params.roomId, + sutUserId: context.sutUserId, + token, + }) && event.relatesTo === undefined, + roomId: params.roomId, + timeoutMs: context.timeoutMs, + }); + const reply = buildMatrixE2eeReplyArtifact(matched.event, token); + assertTopLevelReplyArtifact("E2EE reply", reply); + return { + driverEventId, + driverUserId: params.driverUserId, + reply, + roomId: params.roomId, + roomKey: params.roomKey, + since: matched.since ?? startSince, + token, + }; +} + +export async function runMatrixQaE2eeTopLevelScenario( + context: MatrixQaScenarioContext, + params: { + scenarioId: MatrixQaE2eeScenarioId; + tokenPrefix: string; + }, +) { + const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom(context, params.scenarioId); + return await withMatrixQaE2eeDriver(context, params.scenarioId, async (client) => { + return await runMatrixQaE2eeTopLevelWithClient(context, { + client, + driverUserId: context.driverUserId, + roomId, + roomKey, + tokenPrefix: params.tokenPrefix, + }); + }); +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-shared.ts new file mode 100644 index 000000000000..4ec0d6919a79 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-shared.ts @@ -0,0 +1,422 @@ +// Qa Matrix plugin module implements shared scenario runtime E2EE behavior. +import { randomUUID } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; +import type { MatrixVerificationSummary } from "@openclaw/matrix/test-api.js"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { + createMatrixQaE2eeScenarioClient, + runMatrixQaE2eeBootstrap, +} from "../../substrate/e2ee-client.js"; +import type { MatrixQaE2eeScenarioClient } from "../../substrate/e2ee-client.js"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { + buildMatrixQaE2eeScenarioRoomKey, + type MatrixQaE2eeScenarioId, + resolveMatrixQaScenarioRoomId, +} from "./scenario-catalog.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; + +export const MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT = "/_matrix/client/v3/room_keys/version"; +export const MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID = "room-key-backup-version-unavailable"; +export const MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID = "owner-signature-upload-blocked"; +export const MATRIX_QA_KEYS_SIGNATURES_UPLOAD_ENDPOINT = + "/_matrix/client/v3/keys/signatures/upload"; +export const MATRIX_QA_SYNC_ENDPOINT = "/_matrix/client/v3/sync"; +export const MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID = "sync-state-after-missing-encryption"; +export const MATRIX_QA_SYNC_STATE_AFTER_KEY = "org.matrix.msc4222.state_after"; +export const MATRIX_QA_SYNC_STATE_AFTER_PARAM = "org.matrix.msc4222.use_state_after"; + +type MatrixQaE2eeBootstrapResult = Awaited>; + +export function requireMatrixQaE2eeOutputDir(context: MatrixQaScenarioContext) { + if (!context.outputDir) { + throw new Error("Matrix E2EE QA scenarios require an output directory"); + } + return context.outputDir; +} + +export function requireMatrixQaCliRuntimeEnv(context: MatrixQaScenarioContext) { + if (!context.gatewayRuntimeEnv) { + throw new Error("Matrix CLI QA scenarios require the gateway runtime environment"); + } + return context.gatewayRuntimeEnv; +} + +export function requireMatrixQaGatewayConfigPath(context: MatrixQaScenarioContext) { + const configPath = requireMatrixQaCliRuntimeEnv(context).OPENCLAW_CONFIG_PATH?.trim(); + if (!configPath) { + throw new Error("Matrix CLI QA scenarios require the gateway config path"); + } + return configPath; +} + +export function requireMatrixQaRegistrationToken(context: MatrixQaScenarioContext) { + const token = context.registrationToken?.trim(); + if (!token) { + throw new Error("Matrix CLI QA scenarios require the homeserver registration token"); + } + return token; +} + +export function requireMatrixQaPassword( + context: MatrixQaScenarioContext, + actor: "driver" | "observer" | "sut", +) { + const password = + actor === "driver" + ? context.driverPassword + : actor === "observer" + ? context.observerPassword + : context.sutPassword; + if (!password) { + throw new Error(`Matrix E2EE ${actor} password is required for this scenario`); + } + return password; +} + +export function resolveMatrixQaE2eeScenarioGroupRoom( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, +) { + const roomKey = buildMatrixQaE2eeScenarioRoomKey(scenarioId); + return { + roomKey, + roomId: resolveMatrixQaScenarioRoomId(context, roomKey), + }; +} + +export async function registerMatrixQaE2eeScenarioAccount(params: { + context: MatrixQaScenarioContext; + deviceName: string; + localpartPrefix: string; + scenarioId: MatrixQaE2eeScenarioId; +}) { + const localpartSuffix = params.scenarioId + .replace(/^matrix-e2ee-/, "") + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 24); + const account = await createMatrixQaClient({ + baseUrl: params.context.baseUrl, + }).registerWithToken({ + deviceName: params.deviceName, + localpart: `${params.localpartPrefix}-${localpartSuffix}-${randomUUID().replaceAll("-", "").slice(0, 8)}`, + password: `matrix-qa-${randomUUID()}`, + registrationToken: requireMatrixQaRegistrationToken(params.context), + }); + if (!account.deviceId) { + throw new Error( + `Matrix E2EE QA registration for ${params.scenarioId} did not return a device id`, + ); + } + return account; +} + +export function assertMatrixQaBootstrapSucceeded( + label: string, + result: MatrixQaE2eeBootstrapResult, +) { + if (!result.success) { + throw new Error(`${label} bootstrap failed: ${result.error ?? "unknown error"}`); + } + if (!result.verification.verified || !result.verification.signedByOwner) { + throw new Error(`${label} bootstrap did not leave the device verified by its owner`); + } + if (!result.verification.crossSigningVerified) { + throw new Error(`${label} bootstrap did not establish full Matrix identity trust`); + } + if (!result.crossSigning.published) { + throw new Error(`${label} bootstrap did not publish cross-signing keys`); + } + if (!result.verification.recoveryKeyStored) { + throw new Error(`${label} bootstrap did not store a recovery key`); + } + if (!result.verification.backupVersion) { + throw new Error(`${label} bootstrap did not create a room-key backup`); + } +} + +function isMatrixQaRepairableBackupBootstrapError(error: string | undefined) { + const normalized = error?.toLowerCase() ?? ""; + return ( + normalized.includes("room key backup is not usable") || + normalized.includes("m.megolm_backup.v1") || + normalized.includes("backup decryption key could not be loaded") + ); +} + +const MATRIX_QA_PRESERVE_IDENTITY_BOOTSTRAP_OPTIONS = { + allowAutomaticCrossSigningReset: false, +} as const; + +export async function assertMatrixQaPeerDeviceTrusted(params: { + client: MatrixQaE2eeScenarioClient; + deviceId: string; + label: string; + timeoutMs: number; + userId: string; +}) { + const startedAt = Date.now(); + let status = await params.client.getDeviceVerificationStatus(params.userId, params.deviceId); + while (!status.verified && Date.now() - startedAt < params.timeoutMs) { + await sleep(Math.min(250, Math.max(25, params.timeoutMs - (Date.now() - startedAt)))); + status = await params.client.getDeviceVerificationStatus(params.userId, params.deviceId); + } + if (!status.verified) { + throw new Error( + `${params.label} did not trust ${params.userId}/${params.deviceId} after verification`, + ); + } + return status; +} + +export async function ensureMatrixQaE2eeOwnDeviceVerified(params: { + client: MatrixQaE2eeScenarioClient; + label: string; +}) { + let bootstrap = await params.client.bootstrapOwnDeviceVerification( + MATRIX_QA_PRESERVE_IDENTITY_BOOTSTRAP_OPTIONS, + ); + if (!bootstrap.success && isMatrixQaRepairableBackupBootstrapError(bootstrap.error)) { + const reset = await params.client.resetRoomKeyBackup(); + if (reset.success) { + bootstrap = await params.client.bootstrapOwnDeviceVerification( + MATRIX_QA_PRESERVE_IDENTITY_BOOTSTRAP_OPTIONS, + ); + } + } + assertMatrixQaBootstrapSucceeded(params.label, bootstrap); + return { + bootstrap, + recoveryKey: await params.client.getRecoveryKey(), + verification: bootstrap.verification, + }; +} + +export async function waitForMatrixQaNonEmptyRoomKeyRestore(params: { + client: MatrixQaE2eeScenarioClient; + recoveryKey: string; + timeoutMs: number; +}) { + const startedAt = Date.now(); + let last: Awaited> | null = null; + while (Date.now() - startedAt < params.timeoutMs) { + const restored = await params.client.restoreRoomKeyBackup({ + recoveryKey: params.recoveryKey, + }); + last = restored; + if (!restored.success) { + throw new Error( + `Matrix E2EE room-key backup restore failed: ${restored.error ?? "unknown error"}`, + ); + } + if (restored.total > 0 && restored.imported > 0) { + return restored; + } + await sleep(500); + } + throw new Error( + `Matrix E2EE room-key backup restore did not import any keys before timeout (last imported/total: ${last?.imported ?? 0}/${last?.total ?? 0})`, + ); +} + +export async function waitForMatrixQaVerificationSummary(params: { + client: MatrixQaE2eeScenarioClient; + label: string; + predicate: (summary: MatrixVerificationSummary) => boolean; + timeoutMs: number; +}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < params.timeoutMs) { + const summaries = await params.client.listVerifications(); + const found = summaries.find(params.predicate); + if (found) { + return found; + } + await sleep(Math.min(250, Math.max(25, params.timeoutMs - (Date.now() - startedAt)))); + } + throw new Error(`timed out waiting for Matrix verification summary: ${params.label}`); +} + +export function sameMatrixQaVerificationTransaction( + left: MatrixVerificationSummary, + right: MatrixVerificationSummary, +) { + return Boolean(left.transactionId && left.transactionId === right.transactionId); +} + +export function formatMatrixQaSasEmoji(summary: MatrixVerificationSummary) { + return summary.sas?.emoji?.map(([emoji, label]) => `${emoji} ${label}`) ?? []; +} + +function assertMatrixQaSasEmojiMatches(params: { + initiator: MatrixVerificationSummary; + recipient: MatrixVerificationSummary; +}) { + const initiatorEmoji = formatMatrixQaSasEmoji(params.initiator); + const recipientEmoji = formatMatrixQaSasEmoji(params.recipient); + if (initiatorEmoji.length === 0 || recipientEmoji.length === 0) { + throw new Error("Matrix SAS verification did not expose emoji data on both devices"); + } + if (JSON.stringify(initiatorEmoji) !== JSON.stringify(recipientEmoji)) { + throw new Error("Matrix SAS emoji did not match between verification devices"); + } + return initiatorEmoji; +} + +export function isMatrixQaE2eeNoticeTriggeredSutReply(params: { + event: MatrixQaObservedEvent; + noticeEventId: string; + noticeSentAt: number; + roomId: string; + sutUserId: string; + token: string; +}) { + if ( + params.event.roomId !== params.roomId || + params.event.sender !== params.sutUserId || + params.event.type !== "m.room.message" + ) { + return false; + } + if (params.event.body?.includes(params.token)) { + return true; + } + if ( + params.event.relatesTo?.eventId === params.noticeEventId || + params.event.relatesTo?.inReplyToId === params.noticeEventId + ) { + return true; + } + return ( + typeof params.event.originServerTs === "number" && + params.event.originServerTs >= params.noticeSentAt + ); +} + +export async function createMatrixQaE2eeDriverClient( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, + opts: { actorId?: "driver" | `driver-${string}` } = {}, +) { + return await createMatrixQaE2eeScenarioClient({ + accessToken: context.driverAccessToken, + actorId: opts.actorId ?? "driver", + baseUrl: context.baseUrl, + deviceId: context.driverDeviceId, + observedEvents: context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(context), + password: context.driverPassword, + scenarioId, + timeoutMs: context.timeoutMs, + userId: context.driverUserId, + }); +} + +async function createMatrixQaE2eeObserverClient( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, +) { + return await createMatrixQaE2eeScenarioClient({ + accessToken: context.observerAccessToken, + actorId: "observer", + baseUrl: context.baseUrl, + deviceId: context.observerDeviceId, + observedEvents: context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(context), + password: context.observerPassword, + scenarioId, + timeoutMs: context.timeoutMs, + userId: context.observerUserId, + }); +} + +export async function withMatrixQaE2eeDriverAndObserver( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, + run: (clients: { + driver: MatrixQaE2eeScenarioClient; + observer: MatrixQaE2eeScenarioClient; + }) => Promise, +) { + const driver = await createMatrixQaE2eeDriverClient(context, scenarioId); + const observer = await createMatrixQaE2eeObserverClient(context, scenarioId); + try { + return await run({ driver, observer }); + } finally { + await Promise.all([driver.stop(), observer.stop()]); + } +} + +export async function completeMatrixQaSasVerification(params: { + initiator: MatrixQaE2eeScenarioClient; + recipient: MatrixQaE2eeScenarioClient; + recipientUserId: string; + request: { + deviceId?: string; + roomId?: string; + userId: string; + }; + timeoutMs: number; +}) { + const initiated = await params.initiator.requestVerification(params.request); + const recipientRequested = await waitForMatrixQaVerificationSummary({ + client: params.recipient, + label: "recipient request", + predicate: (summary) => + !summary.initiatedByMe && + (sameMatrixQaVerificationTransaction(summary, initiated) || + (summary.otherUserId !== params.recipientUserId && summary.pending)), + timeoutMs: params.timeoutMs, + }); + if (recipientRequested.canAccept) { + await params.recipient.acceptVerification(recipientRequested.id); + } + await waitForMatrixQaVerificationSummary({ + client: params.initiator, + label: "initiator ready", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.phaseName === "ready", + timeoutMs: params.timeoutMs, + }); + await params.initiator.startVerification(initiated.id, "sas"); + const initiatorSas = await waitForMatrixQaVerificationSummary({ + client: params.initiator, + label: "initiator SAS", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.hasSas, + timeoutMs: params.timeoutMs, + }); + const recipientSas = await waitForMatrixQaVerificationSummary({ + client: params.recipient, + label: "recipient SAS", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiatorSas) && summary.hasSas, + timeoutMs: params.timeoutMs, + }); + const sasEmoji = assertMatrixQaSasEmojiMatches({ + initiator: initiatorSas, + recipient: recipientSas, + }); + await params.initiator.confirmVerificationSas(initiatorSas.id); + await params.recipient.confirmVerificationSas(recipientSas.id); + const completedInitiator = await waitForMatrixQaVerificationSummary({ + client: params.initiator, + label: "initiator complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.completed, + timeoutMs: params.timeoutMs, + }); + const completedRecipient = await waitForMatrixQaVerificationSummary({ + client: params.recipient, + label: "recipient complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, completedInitiator) && summary.completed, + timeoutMs: params.timeoutMs, + }); + return { + completedInitiator, + completedRecipient, + sasEmoji, + }; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-verification.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-verification.ts new file mode 100644 index 000000000000..2eec222a65d7 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee-verification.ts @@ -0,0 +1,312 @@ +// Qa Matrix plugin module implements verification scenario runtime E2EE behavior. +import { createMatrixQaClient } from "../../substrate/client.js"; +import { + MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY, + resolveMatrixQaScenarioRoomId, +} from "./scenario-catalog.js"; +import { withMatrixQaE2eeDriver } from "./scenario-runtime-e2ee-room.js"; +import { + assertMatrixQaPeerDeviceTrusted, + completeMatrixQaSasVerification, + ensureMatrixQaE2eeOwnDeviceVerified, + requireMatrixQaPassword, + sameMatrixQaVerificationTransaction, + waitForMatrixQaVerificationSummary, + withMatrixQaE2eeDriverAndObserver, +} from "./scenario-runtime-e2ee-shared.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runMatrixQaE2eeDeviceSasVerificationScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + requireMatrixQaPassword(context, "observer"); + if (!context.observerDeviceId) { + throw new Error("Matrix E2EE observer device id is required for device SAS verification"); + } + if (!context.driverDeviceId) { + throw new Error("Matrix E2EE driver device id is required for device SAS verification"); + } + const observerDeviceId = context.observerDeviceId; + const driverDeviceId = context.driverDeviceId; + return await withMatrixQaE2eeDriverAndObserver( + context, + "matrix-e2ee-device-sas-verification", + async ({ driver, observer }) => { + await Promise.all([ + ensureMatrixQaE2eeOwnDeviceVerified({ + client: driver, + label: "driver", + }), + ensureMatrixQaE2eeOwnDeviceVerified({ + client: observer, + label: "observer", + }), + ]); + const result = await completeMatrixQaSasVerification({ + initiator: driver, + recipient: observer, + recipientUserId: context.observerUserId, + request: { + deviceId: observerDeviceId, + userId: context.observerUserId, + }, + timeoutMs: context.timeoutMs, + }); + const driverTrust = await assertMatrixQaPeerDeviceTrusted({ + client: driver, + deviceId: observerDeviceId, + label: "driver", + timeoutMs: context.timeoutMs, + userId: context.observerUserId, + }); + const observerTrust = await assertMatrixQaPeerDeviceTrusted({ + client: observer, + deviceId: driverDeviceId, + label: "observer", + timeoutMs: context.timeoutMs, + userId: context.driverUserId, + }); + return { + artifacts: { + completedVerificationIds: [result.completedInitiator.id, result.completedRecipient.id], + currentDeviceId: driverDeviceId, + driverTrustsObserverDevice: driverTrust.verified, + observerTrustsDriverDevice: observerTrust.verified, + sasEmoji: result.sasEmoji, + secondaryDeviceId: observerDeviceId, + }, + details: [ + "driver-to-observer device verification completed with real SAS", + `initiator transaction: ${result.completedInitiator.transactionId ?? ""}`, + `recipient transaction: ${result.completedRecipient.transactionId ?? ""}`, + `driver trusts observer device: ${driverTrust.verified ? "yes" : "no"}`, + `observer trusts driver device: ${observerTrust.verified ? "yes" : "no"}`, + `emoji: ${result.sasEmoji.join(", ")}`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeQrVerificationScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + requireMatrixQaPassword(context, "observer"); + if (!context.observerDeviceId) { + throw new Error("Matrix E2EE observer device id is required for QR verification"); + } + if (!context.driverDeviceId) { + throw new Error("Matrix E2EE driver device id is required for QR verification"); + } + const observerDeviceId = context.observerDeviceId; + const driverDeviceId = context.driverDeviceId; + return await withMatrixQaE2eeDriverAndObserver( + context, + "matrix-e2ee-qr-verification", + async ({ driver, observer }) => { + await Promise.all([ + ensureMatrixQaE2eeOwnDeviceVerified({ + client: driver, + label: "driver", + }), + ensureMatrixQaE2eeOwnDeviceVerified({ + client: observer, + label: "observer", + }), + ]); + const initiated = await driver.requestVerification({ + deviceId: observerDeviceId, + userId: context.observerUserId, + }); + const incoming = await waitForMatrixQaVerificationSummary({ + client: observer, + label: "QR recipient request", + predicate: (summary) => + !summary.initiatedByMe && sameMatrixQaVerificationTransaction(summary, initiated), + timeoutMs: context.timeoutMs, + }); + if (incoming.canAccept) { + await observer.acceptVerification(incoming.id); + } + await waitForMatrixQaVerificationSummary({ + client: driver, + label: "QR request ready", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.phaseName === "ready", + timeoutMs: context.timeoutMs, + }); + const qr = await driver.generateVerificationQr(initiated.id); + await observer.scanVerificationQr(incoming.id, qr.qrDataBase64); + const reciprocate = await waitForMatrixQaVerificationSummary({ + client: driver, + label: "QR reciprocate", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.hasReciprocateQr, + timeoutMs: context.timeoutMs, + }); + await driver.confirmVerificationReciprocateQr(reciprocate.id); + const qrByteCount = Buffer.from(qr.qrDataBase64, "base64").byteLength; + const completedDriver = await waitForMatrixQaVerificationSummary({ + client: driver, + label: "QR driver complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.completed, + timeoutMs: context.timeoutMs, + }); + const completedObserver = await waitForMatrixQaVerificationSummary({ + client: observer, + label: "QR observer complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, completedDriver) && summary.completed, + timeoutMs: context.timeoutMs, + }); + const driverTrust = await assertMatrixQaPeerDeviceTrusted({ + client: driver, + deviceId: observerDeviceId, + label: "driver", + timeoutMs: context.timeoutMs, + userId: context.observerUserId, + }); + const observerTrust = await assertMatrixQaPeerDeviceTrusted({ + client: observer, + deviceId: driverDeviceId, + label: "observer", + timeoutMs: context.timeoutMs, + userId: context.driverUserId, + }); + return { + artifacts: { + completedVerificationIds: [completedDriver.id, completedObserver.id], + driverTrustsObserverDevice: driverTrust.verified, + identityVerificationCompleted: true, + observerTrustsDriverDevice: observerTrust.verified, + qrBytes: qrByteCount, + secondaryDeviceId: observerDeviceId, + }, + details: [ + "driver-to-observer QR verification completed through real QR scan", + `transaction: ${completedDriver.transactionId ?? ""}`, + `driver trusts observer device: ${driverTrust.verified ? "yes" : "no"}`, + `observer trusts driver device: ${observerTrust.verified ? "yes" : "no"}`, + `qr bytes: ${qrByteCount}`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeStaleDeviceHygieneScenario( + context: MatrixQaScenarioContext, +): Promise { + const driverPassword = requireMatrixQaPassword(context, "driver"); + return await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-stale-device-hygiene", + async (client) => { + await ensureMatrixQaE2eeOwnDeviceVerified({ + client, + label: "driver", + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const secondary = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA Stale Device", + password: driverPassword, + userId: context.driverUserId, + }); + if (!secondary.deviceId) { + throw new Error("Matrix stale-device login did not return a secondary device id"); + } + const before = await client.listOwnDevices(); + if (!before.some((device) => device.deviceId === secondary.deviceId)) { + throw new Error("Matrix stale-device list did not include the secondary login"); + } + await client.stop().catch(() => undefined); + const deleted = await client.deleteOwnDevices([secondary.deviceId]); + const remainingDeviceIds = deleted.remainingDevices.map((device) => device.deviceId); + if (remainingDeviceIds.includes(secondary.deviceId)) { + throw new Error( + "Matrix stale-device deletion left the secondary device in the device list", + ); + } + if ( + deleted.currentDeviceId && + !deleted.remainingDevices.some((device) => device.deviceId === deleted.currentDeviceId) + ) { + throw new Error("Matrix stale-device deletion removed the current device"); + } + return { + artifacts: { + currentDeviceId: deleted.currentDeviceId, + deletedDeviceIds: deleted.deletedDeviceIds, + remainingDeviceIds, + secondaryDeviceId: secondary.deviceId, + }, + details: [ + "driver secondary device was created, observed, and removed through real device APIs", + `current device: ${deleted.currentDeviceId ?? ""}`, + `deleted device: ${secondary.deviceId}`, + `remaining devices: ${remainingDeviceIds.join(", ")}`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeDmSasVerificationScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + requireMatrixQaPassword(context, "observer"); + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY); + return await withMatrixQaE2eeDriverAndObserver( + context, + "matrix-e2ee-dm-sas-verification", + async ({ driver, observer }) => { + await Promise.all([ + ensureMatrixQaE2eeOwnDeviceVerified({ + client: driver, + label: "driver", + }), + ensureMatrixQaE2eeOwnDeviceVerified({ + client: observer, + label: "observer", + }), + ]); + const result = await completeMatrixQaSasVerification({ + initiator: driver, + recipient: observer, + recipientUserId: context.observerUserId, + request: { + roomId, + userId: context.observerUserId, + }, + timeoutMs: context.timeoutMs, + }); + if ( + result.completedInitiator.roomId !== roomId || + result.completedRecipient.roomId !== roomId + ) { + throw new Error("Matrix E2EE DM verification completed outside the expected DM room"); + } + return { + artifacts: { + completedVerificationIds: [result.completedInitiator.id, result.completedRecipient.id], + roomKey: MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY, + sasEmoji: result.sasEmoji, + verificationRoomId: roomId, + }, + details: [ + "driver/observer encrypted DM verification completed with SAS in the expected room", + `verification DM room: ${roomId}`, + `transaction: ${result.completedInitiator.transactionId ?? ""}`, + `emoji: ${result.sasEmoji.join(", ")}`, + ].join("\n"), + }; + }, + ); +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts deleted file mode 100644 index 9242475fa42d..000000000000 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts +++ /dev/null @@ -1,3670 +0,0 @@ -// Qa Matrix plugin module implements scenario runtime e2ee behavior. -import { randomUUID } from "node:crypto"; -import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { setTimeout as sleep } from "node:timers/promises"; -import type { MatrixVerificationSummary } from "@openclaw/matrix/test-api.js"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; -import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; -import { createMatrixQaClient } from "../../substrate/client.js"; -import { - createMatrixQaE2eeScenarioClient, - runMatrixQaE2eeBootstrap, -} from "../../substrate/e2ee-client.js"; -import type { MatrixQaE2eeScenarioClient } from "../../substrate/e2ee-client.js"; -import type { MatrixQaObservedEvent } from "../../substrate/events.js"; -import { - startMatrixQaFaultProxy, - type MatrixQaFaultProxyHit, - type MatrixQaFaultProxyRule, -} from "../../substrate/fault-proxy.js"; -import { - buildMatrixQaE2eeScenarioRoomKey, - type MatrixQaE2eeScenarioId, - MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY, - resolveMatrixQaScenarioRoomId, -} from "./scenario-catalog.js"; -import { - buildMatrixQaImageUnderstandingPrompt, - createMatrixQaSplitColorImagePng, - hasMatrixQaExpectedColorReply, - MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, -} from "./scenario-media-fixtures.js"; -import { - formatMatrixQaCliCommand, - redactMatrixQaCliOutput, - runMatrixQaOpenClawCli, - startMatrixQaOpenClawCli, - type MatrixQaCliSession, - type MatrixQaCliRunResult, -} from "./scenario-runtime-cli.js"; -import { - isMatrixQaPlainRecord, - patchMatrixQaGatewayMatrixAccount, - readMatrixQaGatewayMatrixAccount, - replaceMatrixQaGatewayMatrixAccount, -} from "./scenario-runtime-config.js"; -import { - assertThreadReplyArtifact, - assertTopLevelReplyArtifact, - buildMatrixQaToken, - buildMatrixReplyDetails, - buildMentionPrompt, - doesMatrixQaReplyBodyMatchToken, - isMatrixQaExactMarkerReply, - resolveMatrixQaNoReplyWindowMs, - type MatrixQaScenarioContext, -} from "./scenario-runtime-shared.js"; -import type { MatrixQaReplyArtifact, MatrixQaScenarioExecution } from "./scenario-types.js"; - -const MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT = "/_matrix/client/v3/room_keys/version"; -const MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID = "room-key-backup-version-unavailable"; -const MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID = "owner-signature-upload-blocked"; -const MATRIX_QA_KEYS_SIGNATURES_UPLOAD_ENDPOINT = "/_matrix/client/v3/keys/signatures/upload"; -const MATRIX_QA_SYNC_ENDPOINT = "/_matrix/client/v3/sync"; -const MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID = "sync-state-after-missing-encryption"; -const MATRIX_QA_SYNC_STATE_AFTER_KEY = "org.matrix.msc4222.state_after"; -const MATRIX_QA_SYNC_STATE_AFTER_PARAM = "org.matrix.msc4222.use_state_after"; - -type MatrixQaE2eeBootstrapResult = Awaited>; -type MatrixQaCliVerificationStatus = { - backup?: { - decryptionKeyCached?: boolean | null; - keyLoadError?: string | null; - matchesDecryptionKey?: boolean | null; - trusted?: boolean | null; - }; - backupVersion?: string | null; - crossSigningVerified?: boolean; - encryptionEnabled?: boolean; - pendingVerifications?: number; - recoveryKeyStored?: boolean; - serverDeviceKnown?: boolean; - verified?: boolean; - signedByOwner?: boolean; - deviceId?: string | null; - userId?: string | null; -}; -type MatrixQaCliEncryptionSetupStatus = { - accountId?: string; - bootstrap?: { - error?: string; - success?: boolean; - }; - configPath?: string; - encryptionChanged?: boolean; - status?: MatrixQaCliVerificationStatus; - success?: boolean; -}; -type MatrixQaCliAccountAddStatus = { - accountId?: string; - configPath?: string; - encryptionEnabled?: boolean; - verificationBootstrap?: { - attempted?: boolean; - backupVersion?: string | null; - error?: string; - success?: boolean; - }; -}; -type MatrixQaCliBackupRestoreStatus = { - success?: boolean; - backup?: MatrixQaCliVerificationStatus["backup"]; - error?: string; -}; - -function isMatrixQaCliBackupUsable( - backup: MatrixQaCliVerificationStatus["backup"], - opts: { allowUntrustedMatchingKey?: boolean } = {}, -): boolean { - return Boolean( - (backup?.trusted || opts.allowUntrustedMatchingKey === true) && - backup?.matchesDecryptionKey && - backup.decryptionKeyCached && - !backup.keyLoadError, - ); -} - -function requireMatrixQaE2eeOutputDir(context: MatrixQaScenarioContext) { - if (!context.outputDir) { - throw new Error("Matrix E2EE QA scenarios require an output directory"); - } - return context.outputDir; -} - -function requireMatrixQaCliRuntimeEnv(context: MatrixQaScenarioContext) { - if (!context.gatewayRuntimeEnv) { - throw new Error("Matrix CLI QA scenarios require the gateway runtime environment"); - } - return context.gatewayRuntimeEnv; -} - -function requireMatrixQaGatewayConfigPath(context: MatrixQaScenarioContext) { - const configPath = requireMatrixQaCliRuntimeEnv(context).OPENCLAW_CONFIG_PATH?.trim(); - if (!configPath) { - throw new Error("Matrix CLI QA scenarios require the gateway config path"); - } - return configPath; -} - -function requireMatrixQaRegistrationToken(context: MatrixQaScenarioContext) { - const token = context.registrationToken?.trim(); - if (!token) { - throw new Error("Matrix CLI QA scenarios require the homeserver registration token"); - } - return token; -} - -function requireMatrixQaPassword( - context: MatrixQaScenarioContext, - actor: "driver" | "observer" | "sut", -) { - const password = - actor === "driver" - ? context.driverPassword - : actor === "observer" - ? context.observerPassword - : context.sutPassword; - if (!password) { - throw new Error(`Matrix E2EE ${actor} password is required for this scenario`); - } - return password; -} - -function resolveMatrixQaE2eeScenarioGroupRoom( - context: MatrixQaScenarioContext, - scenarioId: MatrixQaE2eeScenarioId, -) { - const roomKey = buildMatrixQaE2eeScenarioRoomKey(scenarioId); - return { - roomKey, - roomId: resolveMatrixQaScenarioRoomId(context, roomKey), - }; -} - -function assertMatrixQaBootstrapSucceeded(label: string, result: MatrixQaE2eeBootstrapResult) { - if (!result.success) { - throw new Error(`${label} bootstrap failed: ${result.error ?? "unknown error"}`); - } - if (!result.verification.verified || !result.verification.signedByOwner) { - throw new Error(`${label} bootstrap did not leave the device verified by its owner`); - } - if (!result.verification.crossSigningVerified) { - throw new Error(`${label} bootstrap did not establish full Matrix identity trust`); - } - if (!result.crossSigning.published) { - throw new Error(`${label} bootstrap did not publish cross-signing keys`); - } - if (!result.verification.recoveryKeyStored) { - throw new Error(`${label} bootstrap did not store a recovery key`); - } - if (!result.verification.backupVersion) { - throw new Error(`${label} bootstrap did not create a room-key backup`); - } -} - -function isMatrixQaRepairableBackupBootstrapError(error: string | undefined) { - const normalized = error?.toLowerCase() ?? ""; - return ( - normalized.includes("room key backup is not usable") || - normalized.includes("m.megolm_backup.v1") || - normalized.includes("backup decryption key could not be loaded") - ); -} - -const MATRIX_QA_PRESERVE_IDENTITY_BOOTSTRAP_OPTIONS = { - allowAutomaticCrossSigningReset: false, -} as const; - -async function assertMatrixQaPeerDeviceTrusted(params: { - client: MatrixQaE2eeScenarioClient; - deviceId: string; - label: string; - timeoutMs: number; - userId: string; -}) { - const startedAt = Date.now(); - let status = await params.client.getDeviceVerificationStatus(params.userId, params.deviceId); - while (!status.verified && Date.now() - startedAt < params.timeoutMs) { - await sleep(Math.min(250, Math.max(25, params.timeoutMs - (Date.now() - startedAt)))); - status = await params.client.getDeviceVerificationStatus(params.userId, params.deviceId); - } - if (!status.verified) { - throw new Error( - `${params.label} did not trust ${params.userId}/${params.deviceId} after verification`, - ); - } - return status; -} - -async function ensureMatrixQaE2eeOwnDeviceVerified(params: { - client: MatrixQaE2eeScenarioClient; - label: string; -}) { - let bootstrap = await params.client.bootstrapOwnDeviceVerification( - MATRIX_QA_PRESERVE_IDENTITY_BOOTSTRAP_OPTIONS, - ); - if (!bootstrap.success && isMatrixQaRepairableBackupBootstrapError(bootstrap.error)) { - const reset = await params.client.resetRoomKeyBackup(); - if (reset.success) { - bootstrap = await params.client.bootstrapOwnDeviceVerification( - MATRIX_QA_PRESERVE_IDENTITY_BOOTSTRAP_OPTIONS, - ); - } - } - assertMatrixQaBootstrapSucceeded(params.label, bootstrap); - return { - bootstrap, - recoveryKey: await params.client.getRecoveryKey(), - verification: bootstrap.verification, - }; -} - -async function waitForMatrixQaNonEmptyRoomKeyRestore(params: { - client: MatrixQaE2eeScenarioClient; - recoveryKey: string; - timeoutMs: number; -}) { - const startedAt = Date.now(); - let last: Awaited> | null = null; - while (Date.now() - startedAt < params.timeoutMs) { - const restored = await params.client.restoreRoomKeyBackup({ - recoveryKey: params.recoveryKey, - }); - last = restored; - if (!restored.success) { - throw new Error( - `Matrix E2EE room-key backup restore failed: ${restored.error ?? "unknown error"}`, - ); - } - if (restored.total > 0 && restored.imported > 0) { - return restored; - } - await sleep(500); - } - throw new Error( - `Matrix E2EE room-key backup restore did not import any keys before timeout (last imported/total: ${last?.imported ?? 0}/${last?.total ?? 0})`, - ); -} - -async function waitForMatrixQaVerificationSummary(params: { - client: MatrixQaE2eeScenarioClient; - label: string; - predicate: (summary: MatrixVerificationSummary) => boolean; - timeoutMs: number; -}) { - const startedAt = Date.now(); - while (Date.now() - startedAt < params.timeoutMs) { - const summaries = await params.client.listVerifications(); - const found = summaries.find(params.predicate); - if (found) { - return found; - } - await sleep(Math.min(250, Math.max(25, params.timeoutMs - (Date.now() - startedAt)))); - } - throw new Error(`timed out waiting for Matrix verification summary: ${params.label}`); -} - -function sameMatrixQaVerificationTransaction( - left: MatrixVerificationSummary, - right: MatrixVerificationSummary, -) { - return Boolean(left.transactionId && left.transactionId === right.transactionId); -} - -function formatMatrixQaSasEmoji(summary: MatrixVerificationSummary) { - return summary.sas?.emoji?.map(([emoji, label]) => `${emoji} ${label}`) ?? []; -} - -function parseMatrixQaCliJsonText(text: string): unknown { - const candidate = text.trim(); - if (!candidate) { - throw new Error("no JSON payload found"); - } - return JSON.parse(candidate) as unknown; -} - -function parseMatrixQaCliJson(result: MatrixQaCliRunResult): unknown { - const stdout = result.stdout.trim(); - const stderr = result.stderr.trim(); - if (stdout) { - try { - return parseMatrixQaCliJsonText(stdout); - } catch (error) { - throw new Error( - `${formatMatrixQaCliCommand(result.args)} printed invalid JSON: ${ - error instanceof Error ? error.message : String(error) - }\nstdout:\n${redactMatrixQaCliOutput(stdout)}`, - { cause: error }, - ); - } - } - - if (!stderr) { - throw new Error(`${formatMatrixQaCliCommand(result.args)} did not print JSON`); - } - try { - return parseMatrixQaCliJsonText(stderr); - } catch (error) { - throw new Error( - `${formatMatrixQaCliCommand(result.args)} printed invalid JSON: ${ - error instanceof Error ? error.message : String(error) - }\nstderr:\n${redactMatrixQaCliOutput(stderr)}`, - { cause: error }, - ); - } -} - -function buildMatrixQaPluginActivationConfig() { - return { - plugins: { - allow: ["matrix"], - entries: { - matrix: { enabled: true }, - }, - }, - }; -} - -function buildMatrixQaEmptyMatrixCliConfig() { - return { - ...buildMatrixQaPluginActivationConfig(), - channels: { - matrix: { - enabled: true, - accounts: {}, - }, - }, - }; -} - -async function registerMatrixQaCliE2eeAccount(params: { - context: MatrixQaScenarioContext; - deviceName: string; - scenarioId: MatrixQaE2eeScenarioId; -}) { - const localpartSuffix = params.scenarioId - .replace(/^matrix-e2ee-cli-/, "") - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 24); - const account = await createMatrixQaClient({ - baseUrl: params.context.baseUrl, - }).registerWithToken({ - deviceName: params.deviceName, - localpart: `qa-cli-${localpartSuffix}-${randomUUID().replaceAll("-", "").slice(0, 8)}`, - password: `matrix-qa-${randomUUID()}`, - registrationToken: requireMatrixQaRegistrationToken(params.context), - }); - if (!account.deviceId) { - throw new Error( - `Matrix CLI QA registration for ${params.scenarioId} did not return a device id`, - ); - } - return account; -} - -async function registerMatrixQaE2eeScenarioAccount(params: { - context: MatrixQaScenarioContext; - deviceName: string; - localpartPrefix: string; - scenarioId: MatrixQaE2eeScenarioId; -}) { - const localpartSuffix = params.scenarioId - .replace(/^matrix-e2ee-/, "") - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 24); - const account = await createMatrixQaClient({ - baseUrl: params.context.baseUrl, - }).registerWithToken({ - deviceName: params.deviceName, - localpart: `${params.localpartPrefix}-${localpartSuffix}-${randomUUID().replaceAll("-", "").slice(0, 8)}`, - password: `matrix-qa-${randomUUID()}`, - registrationToken: requireMatrixQaRegistrationToken(params.context), - }); - if (!account.deviceId) { - throw new Error( - `Matrix E2EE QA registration for ${params.scenarioId} did not return a device id`, - ); - } - return account; -} - -async function createMatrixQaE2eeCliOwnerClient(params: { - account: Awaited>; - context: MatrixQaScenarioContext; - scenarioId: MatrixQaE2eeScenarioId; -}) { - return await createMatrixQaE2eeScenarioClient({ - accessToken: params.account.accessToken, - actorId: `cli-owner-${randomUUID().slice(0, 8)}`, - baseUrl: params.context.baseUrl, - deviceId: params.account.deviceId, - observedEvents: params.context.observedEvents, - outputDir: requireMatrixQaE2eeOutputDir(params.context), - password: params.account.password, - scenarioId: params.scenarioId, - timeoutMs: params.context.timeoutMs, - userId: params.account.userId, - }); -} - -function parseMatrixQaCliSasText( - text: string, - label: string, -): { kind: "emoji"; value: string } | { kind: "decimal"; value: string } { - const emoji = text.match(/^SAS emoji:\s*(.+)$/m)?.[1]?.trim(); - if (emoji) { - return { kind: "emoji", value: emoji }; - } - const decimal = text.match(/^SAS decimals:\s*(.+)$/m)?.[1]?.trim(); - if (decimal) { - return { kind: "decimal", value: decimal }; - } - throw new Error(`${label} did not print SAS emoji or decimals`); -} - -function parseMatrixQaCliSummaryField(text: string, field: string): string | null { - const escaped = field.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - return text.match(new RegExp(`^${escaped}:\\s*(.+)$`, "m"))?.[1]?.trim() ?? null; -} - -async function writeMatrixQaCliOutputArtifacts(params: { - label: string; - result: MatrixQaCliRunResult; - rootDir: string; -}) { - await mkdir(params.rootDir, { mode: 0o700, recursive: true }); - await chmod(params.rootDir, 0o700).catch(() => undefined); - const prefix = params.label.replace(/[^A-Za-z0-9_-]/g, "-"); - const stdoutPath = path.join(params.rootDir, `${prefix}.stdout.txt`); - const stderrPath = path.join(params.rootDir, `${prefix}.stderr.txt`); - await Promise.all([ - writeFile(stdoutPath, redactMatrixQaCliOutput(params.result.stdout), { mode: 0o600 }), - writeFile(stderrPath, redactMatrixQaCliOutput(params.result.stderr), { mode: 0o600 }), - ]); - return { stderrPath, stdoutPath }; -} - -async function assertMatrixQaPrivatePathMode(pathToCheck: string, label: string) { - if (process.platform === "win32") { - return; - } - const mode = (await stat(pathToCheck)).mode & 0o777; - if ((mode & 0o077) !== 0) { - throw new Error(`${label} permissions are too broad: ${mode.toString(8)}`); - } -} - -function assertMatrixQaCliSasMatches(params: { - cliSas: ReturnType; - owner: MatrixVerificationSummary; -}) { - if (params.cliSas.kind === "emoji") { - const ownerEmoji = formatMatrixQaSasEmoji(params.owner).join(" | "); - if (!ownerEmoji) { - throw new Error("Matrix owner client did not expose SAS emoji"); - } - if (params.cliSas.value !== ownerEmoji) { - throw new Error("Matrix CLI SAS emoji did not match the owner client"); - } - return ownerEmoji.split(" | "); - } - - const ownerDecimal = params.owner.sas?.decimal?.join(" "); - if (!ownerDecimal) { - throw new Error("Matrix owner client did not expose SAS decimals"); - } - if (params.cliSas.value !== ownerDecimal) { - throw new Error("Matrix CLI SAS decimals did not match the owner client"); - } - return [ownerDecimal]; -} - -function isMatrixQaCliOwnerSelfVerification(params: { - cliDeviceId?: string; - ownerUserId: string; - requireCompleted?: boolean; - requirePending?: boolean; - requireSas?: boolean; - summary: MatrixVerificationSummary; - transactionId?: string; -}) { - const summary = params.summary; - if ( - !summary.isSelfVerification || - summary.initiatedByMe || - summary.otherUserId !== params.ownerUserId - ) { - return false; - } - if (params.transactionId) { - if (summary.transactionId !== params.transactionId) { - return false; - } - } else if (params.cliDeviceId && summary.otherDeviceId !== params.cliDeviceId) { - return false; - } - if (params.requirePending === true && !summary.pending) { - return false; - } - if (params.requireSas === true && !summary.hasSas) { - return false; - } - return params.requireCompleted !== true || summary.completed; -} - -async function createMatrixQaCliSelfVerificationRuntime(params: { - accountId: string; - accessToken: string; - context: MatrixQaScenarioContext; - deviceId: string; - userId: string; -}) { - const outputDir = requireMatrixQaE2eeOutputDir(params.context); - const rootDir = await mkdtemp( - path.join(resolvePreferredOpenClawTmpDir(), "openclaw-matrix-cli-qa-"), - ); - const artifactDir = path.join( - outputDir, - "cli-self-verification", - randomUUID().replaceAll("-", "").slice(0, 12), - ); - const stateDir = path.join(rootDir, "state"); - const configPath = path.join(rootDir, "config.json"); - await chmod(rootDir, 0o700).catch(() => undefined); - await assertMatrixQaPrivatePathMode(rootDir, "Matrix QA CLI temp directory"); - await mkdir(artifactDir, { mode: 0o700, recursive: true }); - await chmod(artifactDir, 0o700).catch(() => undefined); - await assertMatrixQaPrivatePathMode(artifactDir, "Matrix QA CLI artifact directory"); - await mkdir(stateDir, { mode: 0o700, recursive: true }); - await chmod(stateDir, 0o700).catch(() => undefined); - await assertMatrixQaPrivatePathMode(stateDir, "Matrix QA CLI state directory"); - await writeFile( - configPath, - `${JSON.stringify( - { - plugins: { - allow: ["matrix"], - entries: { - matrix: { enabled: true }, - }, - }, - channels: { - matrix: { - defaultAccount: params.accountId, - accounts: { - [params.accountId]: { - accessToken: params.accessToken, - deviceId: params.deviceId, - encryption: true, - homeserver: params.context.baseUrl, - initialSyncLimit: 0, - name: "Matrix QA CLI self-verification", - network: { - dangerouslyAllowPrivateNetwork: true, - }, - startupVerification: "off", - userId: params.userId, - }, - }, - }, - }, - }, - null, - 2, - )}\n`, - { flag: "wx", mode: 0o600 }, - ); - await assertMatrixQaPrivatePathMode(configPath, "Matrix QA CLI config file"); - const env = { - ...requireMatrixQaCliRuntimeEnv(params.context), - FORCE_COLOR: "0", - NO_COLOR: "1", - OPENCLAW_CONFIG_PATH: configPath, - OPENCLAW_DISABLE_AUTO_UPDATE: "1", - OPENCLAW_STATE_DIR: stateDir, - }; - const run = async (args: string[], timeoutMs = params.context.timeoutMs, stdin?: string) => - await runMatrixQaOpenClawCli({ - args, - env, - stdin, - timeoutMs, - }); - const start = (args: string[], timeoutMs = params.context.timeoutMs) => - startMatrixQaOpenClawCli({ - args, - env, - timeoutMs, - }); - return { - configPath, - dispose: async () => { - await rm(rootDir, { force: true, recursive: true }); - }, - run, - rootDir: artifactDir, - start, - stateDir, - }; -} - -async function createMatrixQaCliE2eeSetupRuntime(params: { - artifactLabel: string; - context: MatrixQaScenarioContext; - initialConfig?: Record; -}) { - const outputDir = requireMatrixQaE2eeOutputDir(params.context); - const rootDir = await mkdtemp( - path.join(resolvePreferredOpenClawTmpDir(), "openclaw-matrix-e2ee-setup-qa-"), - ); - const artifactDir = path.join( - outputDir, - params.artifactLabel, - randomUUID().replaceAll("-", "").slice(0, 12), - ); - const stateDir = path.join(rootDir, "state"); - const configPath = path.join(rootDir, "config.json"); - await chmod(rootDir, 0o700).catch(() => undefined); - await assertMatrixQaPrivatePathMode(rootDir, "Matrix QA CLI temp directory"); - await mkdir(artifactDir, { mode: 0o700, recursive: true }); - await chmod(artifactDir, 0o700).catch(() => undefined); - await assertMatrixQaPrivatePathMode(artifactDir, "Matrix QA CLI artifact directory"); - await mkdir(stateDir, { mode: 0o700, recursive: true }); - await chmod(stateDir, 0o700).catch(() => undefined); - await assertMatrixQaPrivatePathMode(stateDir, "Matrix QA CLI state directory"); - await writeFile( - configPath, - `${JSON.stringify(params.initialConfig ?? buildMatrixQaEmptyMatrixCliConfig(), null, 2)}\n`, - { flag: "wx", mode: 0o600 }, - ); - await assertMatrixQaPrivatePathMode(configPath, "Matrix QA CLI config file"); - const env = { - ...requireMatrixQaCliRuntimeEnv(params.context), - FORCE_COLOR: "0", - NO_COLOR: "1", - OPENCLAW_CONFIG_PATH: configPath, - OPENCLAW_DISABLE_AUTO_UPDATE: "1", - OPENCLAW_STATE_DIR: stateDir, - }; - const run = async (args: string[], timeoutMs = params.context.timeoutMs) => - await runMatrixQaOpenClawCli({ - args, - env, - timeoutMs, - }); - const start = (args: string[], timeoutMs = params.context.timeoutMs) => - startMatrixQaOpenClawCli({ - args, - env, - timeoutMs, - }); - return { - configPath, - dispose: async () => { - await rm(rootDir, { force: true, recursive: true }); - }, - run, - rootDir: artifactDir, - start, - stateDir, - }; -} - -async function createMatrixQaCliGatewayRuntime(params: { - artifactLabel: string; - context: MatrixQaScenarioContext; -}) { - const outputDir = requireMatrixQaE2eeOutputDir(params.context); - const artifactDir = path.join( - outputDir, - params.artifactLabel, - randomUUID().replaceAll("-", "").slice(0, 12), - ); - await mkdir(artifactDir, { mode: 0o700, recursive: true }); - await chmod(artifactDir, 0o700).catch(() => undefined); - await assertMatrixQaPrivatePathMode(artifactDir, "Matrix QA CLI artifact directory"); - const env = { - ...requireMatrixQaCliRuntimeEnv(params.context), - FORCE_COLOR: "0", - NO_COLOR: "1", - OPENCLAW_DISABLE_AUTO_UPDATE: "1", - }; - const run = async (args: string[], timeoutMs = params.context.timeoutMs) => - await runMatrixQaOpenClawCli({ - args, - env, - timeoutMs, - }); - return { - dispose: async () => undefined, - rootDir: artifactDir, - run, - }; -} - -function assertMatrixQaSasEmojiMatches(params: { - initiator: MatrixVerificationSummary; - recipient: MatrixVerificationSummary; -}) { - const initiatorEmoji = formatMatrixQaSasEmoji(params.initiator); - const recipientEmoji = formatMatrixQaSasEmoji(params.recipient); - if (initiatorEmoji.length === 0 || recipientEmoji.length === 0) { - throw new Error("Matrix SAS verification did not expose emoji data on both devices"); - } - if (JSON.stringify(initiatorEmoji) !== JSON.stringify(recipientEmoji)) { - throw new Error("Matrix SAS emoji did not match between verification devices"); - } - return initiatorEmoji; -} - -function isMatrixQaE2eeNoticeTriggeredSutReply(params: { - event: MatrixQaObservedEvent; - noticeEventId: string; - noticeSentAt: number; - roomId: string; - sutUserId: string; - token: string; -}) { - if ( - params.event.roomId !== params.roomId || - params.event.sender !== params.sutUserId || - params.event.type !== "m.room.message" - ) { - return false; - } - if (params.event.body?.includes(params.token)) { - return true; - } - if ( - params.event.relatesTo?.eventId === params.noticeEventId || - params.event.relatesTo?.inReplyToId === params.noticeEventId - ) { - return true; - } - return ( - typeof params.event.originServerTs === "number" && - params.event.originServerTs >= params.noticeSentAt - ); -} - -async function createMatrixQaE2eeDriverClient( - context: MatrixQaScenarioContext, - scenarioId: MatrixQaE2eeScenarioId, - opts: { actorId?: "driver" | `driver-${string}` } = {}, -) { - return await createMatrixQaE2eeScenarioClient({ - accessToken: context.driverAccessToken, - actorId: opts.actorId ?? "driver", - baseUrl: context.baseUrl, - deviceId: context.driverDeviceId, - observedEvents: context.observedEvents, - outputDir: requireMatrixQaE2eeOutputDir(context), - password: context.driverPassword, - scenarioId, - timeoutMs: context.timeoutMs, - userId: context.driverUserId, - }); -} - -async function createMatrixQaE2eeObserverClient( - context: MatrixQaScenarioContext, - scenarioId: MatrixQaE2eeScenarioId, -) { - return await createMatrixQaE2eeScenarioClient({ - accessToken: context.observerAccessToken, - actorId: "observer", - baseUrl: context.baseUrl, - deviceId: context.observerDeviceId, - observedEvents: context.observedEvents, - outputDir: requireMatrixQaE2eeOutputDir(context), - password: context.observerPassword, - scenarioId, - timeoutMs: context.timeoutMs, - userId: context.observerUserId, - }); -} - -async function withMatrixQaE2eeDriverAndObserver( - context: MatrixQaScenarioContext, - scenarioId: MatrixQaE2eeScenarioId, - run: (clients: { - driver: MatrixQaE2eeScenarioClient; - observer: MatrixQaE2eeScenarioClient; - }) => Promise, -) { - const driver = await createMatrixQaE2eeDriverClient(context, scenarioId); - const observer = await createMatrixQaE2eeObserverClient(context, scenarioId); - try { - return await run({ driver, observer }); - } finally { - await Promise.all([driver.stop(), observer.stop()]); - } -} - -async function completeMatrixQaSasVerification(params: { - initiator: MatrixQaE2eeScenarioClient; - recipient: MatrixQaE2eeScenarioClient; - recipientUserId: string; - request: { - deviceId?: string; - roomId?: string; - userId: string; - }; - timeoutMs: number; -}) { - const initiated = await params.initiator.requestVerification(params.request); - const recipientRequested = await waitForMatrixQaVerificationSummary({ - client: params.recipient, - label: "recipient request", - predicate: (summary) => - !summary.initiatedByMe && - (sameMatrixQaVerificationTransaction(summary, initiated) || - (summary.otherUserId !== params.recipientUserId && summary.pending)), - timeoutMs: params.timeoutMs, - }); - if (recipientRequested.canAccept) { - await params.recipient.acceptVerification(recipientRequested.id); - } - await waitForMatrixQaVerificationSummary({ - client: params.initiator, - label: "initiator ready", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, initiated) && summary.phaseName === "ready", - timeoutMs: params.timeoutMs, - }); - await params.initiator.startVerification(initiated.id, "sas"); - const initiatorSas = await waitForMatrixQaVerificationSummary({ - client: params.initiator, - label: "initiator SAS", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, initiated) && summary.hasSas, - timeoutMs: params.timeoutMs, - }); - const recipientSas = await waitForMatrixQaVerificationSummary({ - client: params.recipient, - label: "recipient SAS", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, initiatorSas) && summary.hasSas, - timeoutMs: params.timeoutMs, - }); - const sasEmoji = assertMatrixQaSasEmojiMatches({ - initiator: initiatorSas, - recipient: recipientSas, - }); - await params.initiator.confirmVerificationSas(initiatorSas.id); - await params.recipient.confirmVerificationSas(recipientSas.id); - const completedInitiator = await waitForMatrixQaVerificationSummary({ - client: params.initiator, - label: "initiator complete", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, initiated) && summary.completed, - timeoutMs: params.timeoutMs, - }); - const completedRecipient = await waitForMatrixQaVerificationSummary({ - client: params.recipient, - label: "recipient complete", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, completedInitiator) && summary.completed, - timeoutMs: params.timeoutMs, - }); - return { - completedInitiator, - completedRecipient, - sasEmoji, - }; -} - -function buildMatrixE2eeReplyArtifact( - event: MatrixQaObservedEvent, - token: string, -): MatrixQaReplyArtifact { - return { - eventId: event.eventId, - mentions: event.mentions, - relatesTo: event.relatesTo, - sender: event.sender, - tokenMatched: doesMatrixQaReplyBodyMatchToken(event, token), - }; -} - -function buildRoomKeyBackupUnavailableFaultRule(accessToken: string): MatrixQaFaultProxyRule { - return { - id: MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID, - match: (request) => - request.method === "GET" && - request.path === MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT && - request.bearerToken === accessToken, - response: () => ({ - body: { - errcode: "M_NOT_FOUND", - error: "No current key backup", - }, - status: 404, - }), - }; -} - -function buildOwnerSignatureUploadBlockedFaultRule(accessToken: string): MatrixQaFaultProxyRule { - return { - id: MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID, - match: (request) => - request.method === "POST" && - request.path === MATRIX_QA_KEYS_SIGNATURES_UPLOAD_ENDPOINT && - request.bearerToken === accessToken, - response: () => ({ - body: {}, - status: 200, - }), - }; -} - -function removeMatrixQaSyncStateAfterEncryptionEvents(payload: unknown) { - if (!isMatrixQaPlainRecord(payload)) { - return 0; - } - const rooms = isMatrixQaPlainRecord(payload.rooms) ? payload.rooms : {}; - const join = isMatrixQaPlainRecord(rooms.join) ? rooms.join : {}; - let removed = 0; - for (const room of Object.values(join)) { - if (!isMatrixQaPlainRecord(room)) { - continue; - } - const stateAfter = room[MATRIX_QA_SYNC_STATE_AFTER_KEY]; - if (!isMatrixQaPlainRecord(stateAfter) || !Array.isArray(stateAfter.events)) { - continue; - } - const filtered = stateAfter.events.filter((event) => { - if (isMatrixQaPlainRecord(event) && event.type === "m.room.encryption") { - removed += 1; - return false; - } - return true; - }); - stateAfter.events = filtered; - } - return removed; -} - -function buildSyncStateAfterMissingEncryptionFaultRule( - accessToken: string, -): MatrixQaFaultProxyRule { - return { - id: MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID, - match: (request) => - request.method === "GET" && - request.path === MATRIX_QA_SYNC_ENDPOINT && - request.bearerToken === accessToken && - new URLSearchParams(request.search).get(MATRIX_QA_SYNC_STATE_AFTER_PARAM) === "true", - mutateResponse: ({ response }) => { - const contentType = response.headers.get("content-type") ?? ""; - if (!contentType.includes("json")) { - return response; - } - const payload = JSON.parse(response.body.toString("utf8")) as unknown; - removeMatrixQaSyncStateAfterEncryptionEvents(payload); - return { - ...response, - body: Buffer.from(JSON.stringify(payload)), - }; - }, - }; -} - -async function runMatrixQaFaultedE2eeBootstrap(context: MatrixQaScenarioContext): Promise<{ - faultHits: MatrixQaFaultProxyHit[]; - result: MatrixQaE2eeBootstrapResult; -}> { - const proxy = await startMatrixQaFaultProxy({ - targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl, - ...context.faultProxyObserver, - rules: [buildRoomKeyBackupUnavailableFaultRule(context.driverAccessToken)], - }); - try { - const result = await runMatrixQaE2eeBootstrap({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: proxy.baseUrl, - deviceId: context.driverDeviceId, - outputDir: requireMatrixQaE2eeOutputDir(context), - ...(context.driverPassword ? { password: context.driverPassword } : {}), - scenarioId: "matrix-e2ee-key-bootstrap-failure", - timeoutMs: context.timeoutMs, - userId: context.driverUserId, - }); - return { - faultHits: proxy.hits(), - result, - }; - } finally { - await proxy.stop(); - } -} - -async function runMatrixQaFaultedRecoveryOwnerVerification(params: { - accessToken: string; - context: MatrixQaScenarioContext; - deviceId: string; - encodedRecoveryKey: string; - userId: string; -}): Promise<{ - faultHits: MatrixQaFaultProxyHit[]; - restore: Awaited>; - verification: Awaited>; -}> { - const proxy = await startMatrixQaFaultProxy({ - targetBaseUrl: params.context.faultProxyTargetBaseUrl ?? params.context.baseUrl, - ...params.context.faultProxyObserver, - rules: [buildOwnerSignatureUploadBlockedFaultRule(params.accessToken)], - }); - const recoveryClient = await createMatrixQaE2eeScenarioClient({ - accessToken: params.accessToken, - actorId: `driver-recovery-${randomUUID().slice(0, 8)}`, - baseUrl: proxy.baseUrl, - deviceId: params.deviceId, - observedEvents: params.context.observedEvents, - outputDir: requireMatrixQaE2eeOutputDir(params.context), - scenarioId: "matrix-e2ee-recovery-owner-verification-required", - timeoutMs: params.context.timeoutMs, - userId: params.userId, - }); - try { - const verification = await recoveryClient.verifyWithRecoveryKey(params.encodedRecoveryKey); - const restore = await waitForMatrixQaNonEmptyRoomKeyRestore({ - client: recoveryClient, - recoveryKey: params.encodedRecoveryKey, - timeoutMs: params.context.timeoutMs, - }); - return { - faultHits: proxy.hits(), - restore, - verification, - }; - } finally { - await recoveryClient.stop().catch(() => undefined); - await proxy.stop(); - } -} - -function assertMatrixQaFaultedRecoveryOwnerVerificationRequired( - faulted: Awaited>, -) { - if (faulted.faultHits.length === 0) { - throw new Error("Matrix E2EE owner signature fault proxy was not exercised"); - } - if (faulted.verification.success) { - throw new Error( - "Matrix E2EE recovery verification unexpectedly succeeded while owner signature upload was blocked", - ); - } - if (!faulted.verification.recoveryKeyAccepted) { - throw new Error("Matrix E2EE recovery key was not accepted"); - } - if (!faulted.verification.backupUsable) { - throw new Error("Matrix E2EE recovery key did not leave room-key backup usable"); - } - if (faulted.verification.deviceOwnerVerified) { - throw new Error("Matrix E2EE recovery device should still require Matrix identity trust"); - } - if (!faulted.restore.success) { - throw new Error( - `Matrix E2EE room-key backup restore failed after owner-verification fault: ${faulted.restore.error ?? "unknown error"}`, - ); - } -} - -function assertMatrixQaExpectedBootstrapFailure(params: { - faultHits: MatrixQaFaultProxyHit[]; - result: MatrixQaE2eeBootstrapResult; -}) { - if (params.faultHits.length === 0) { - throw new Error("Matrix E2EE bootstrap fault proxy was not exercised"); - } - if (params.result.success) { - throw new Error( - "Matrix E2EE bootstrap unexpectedly succeeded while room-key backup was faulted", - ); - } - const bootstrapError = params.result.error ?? ""; - if (!bootstrapError.toLowerCase().includes("room key backup")) { - throw new Error(`Matrix E2EE bootstrap failed for an unexpected reason: ${bootstrapError}`); - } - return bootstrapError; -} - -async function withMatrixQaE2eeDriver( - context: MatrixQaScenarioContext, - scenarioId: MatrixQaE2eeScenarioId, - run: (client: MatrixQaE2eeScenarioClient) => Promise, - opts: { actorId?: "driver" | `driver-${string}` } = {}, -) { - const client = await createMatrixQaE2eeDriverClient(context, scenarioId, opts); - try { - return await run(client); - } finally { - await client.stop(); - } -} - -async function createMatrixQaE2eeRegisteredScenarioClient(params: { - account: Awaited>; - actorId: `driver-${string}`; - context: MatrixQaScenarioContext; - scenarioId: MatrixQaE2eeScenarioId; -}) { - return await createMatrixQaE2eeScenarioClient({ - accessToken: params.account.accessToken, - actorId: params.actorId, - baseUrl: params.context.baseUrl, - deviceId: params.account.deviceId, - observedEvents: params.context.observedEvents, - outputDir: requireMatrixQaE2eeOutputDir(params.context), - password: params.account.password, - scenarioId: params.scenarioId, - timeoutMs: params.context.timeoutMs, - userId: params.account.userId, - }); -} - -async function withMatrixQaIsolatedE2eeDriverRoom( - context: MatrixQaScenarioContext, - scenarioId: MatrixQaE2eeScenarioId, - run: (params: { - client: MatrixQaE2eeScenarioClient; - driverUserId: string; - roomId: string; - roomKey: string; - }) => Promise, -) { - if (!context.restartGatewayAfterStateMutation) { - throw new Error( - "Matrix E2EE isolated driver room scenario requires hard gateway restart support", - ); - } - const accountId = context.sutAccountId ?? "sut"; - const configPath = requireMatrixQaGatewayConfigPath(context); - const accountConfig = await readMatrixQaGatewayMatrixAccount({ - accountId, - configPath, - }); - const originalGroups = isMatrixQaPlainRecord(accountConfig.groups) ? accountConfig.groups : {}; - const originalGroupAllowFrom = Array.isArray(accountConfig.groupAllowFrom) - ? accountConfig.groupAllowFrom - : undefined; - const originalGroupPolicy = accountConfig.groupPolicy; - const driverAccount = await registerMatrixQaE2eeScenarioAccount({ - context, - deviceName: "OpenClaw Matrix QA Isolated E2EE Driver", - localpartPrefix: "qa-e2ee-driver", - scenarioId, - }); - const driverApi = createMatrixQaClient({ - accessToken: driverAccount.accessToken, - baseUrl: context.baseUrl, - }); - const roomKey = buildMatrixQaE2eeScenarioRoomKey(scenarioId); - const roomId = await driverApi.createPrivateRoom({ - encrypted: true, - inviteUserIds: [context.observerUserId, context.sutUserId], - name: `Matrix QA ${scenarioId} Isolated E2EE Room`, - }); - await Promise.all([ - createMatrixQaClient({ - accessToken: context.observerAccessToken, - baseUrl: context.baseUrl, - }).joinRoom(roomId), - createMatrixQaClient({ - accessToken: context.sutAccessToken, - baseUrl: context.baseUrl, - }).joinRoom(roomId), - ]); - - const isolatedGroups = { - [roomId]: { - enabled: true, - requireMention: true, - }, - }; - const applyPatch = async (accountPatch: Record) => { - await context.restartGatewayAfterStateMutation?.( - async () => { - await patchMatrixQaGatewayMatrixAccount({ - accountId, - accountPatch, - configPath, - }); - }, - { - timeoutMs: context.timeoutMs, - waitAccountId: accountId, - }, - ); - }; - - let patchedGateway; - let client: MatrixQaE2eeScenarioClient | undefined; - try { - await applyPatch({ - groupAllowFrom: [driverAccount.userId], - groupPolicy: "allowlist", - groups: isolatedGroups, - }); - patchedGateway = true; - const actorId: `driver-${string}` = `driver-${scenarioId - .replace(/^matrix-e2ee-/, "") - .replace(/[^A-Za-z0-9_-]/g, "-") - .slice(0, 28)}`; - client = await createMatrixQaE2eeRegisteredScenarioClient({ - account: driverAccount, - actorId, - context, - scenarioId, - }); - await Promise.all([ - client.waitForJoinedMember({ - roomId, - timeoutMs: context.timeoutMs, - userId: context.sutUserId, - }), - client.waitForJoinedMember({ - roomId, - timeoutMs: context.timeoutMs, - userId: context.observerUserId, - }), - ]); - return await run({ - client, - driverUserId: driverAccount.userId, - roomId, - roomKey, - }); - } finally { - await client?.stop().catch(() => undefined); - if (patchedGateway) { - const restorePatch: Record = { - groupAllowFrom: originalGroupAllowFrom, - groupPolicy: originalGroupPolicy, - groups: originalGroups, - }; - await applyPatch(restorePatch).catch(() => undefined); - } - } -} - -async function runMatrixQaE2eeTopLevelWithClient( - context: MatrixQaScenarioContext, - params: { - client: MatrixQaE2eeScenarioClient; - driverUserId: string; - roomId: string; - roomKey: string; - tokenPrefix: string; - }, -) { - const startSince = await params.client.prime(); - const token = buildMatrixQaToken(params.tokenPrefix); - const body = buildMentionPrompt(context.sutUserId, token); - const driverEventId = await params.client.sendTextMessage({ - body, - mentionUserIds: [context.sutUserId], - roomId: params.roomId, - }); - const matched = await params.client.waitForRoomEvent({ - predicate: (event) => - isMatrixQaExactMarkerReply(event, { - roomId: params.roomId, - sutUserId: context.sutUserId, - token, - }) && event.relatesTo === undefined, - roomId: params.roomId, - timeoutMs: context.timeoutMs, - }); - const reply = buildMatrixE2eeReplyArtifact(matched.event, token); - assertTopLevelReplyArtifact("E2EE reply", reply); - return { - driverEventId, - driverUserId: params.driverUserId, - reply, - roomId: params.roomId, - roomKey: params.roomKey, - since: matched.since ?? startSince, - token, - }; -} - -async function runMatrixQaE2eeTopLevelScenario( - context: MatrixQaScenarioContext, - params: { - scenarioId: MatrixQaE2eeScenarioId; - tokenPrefix: string; - }, -) { - const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom(context, params.scenarioId); - return await withMatrixQaE2eeDriver(context, params.scenarioId, async (client) => { - return await runMatrixQaE2eeTopLevelWithClient(context, { - client, - driverUserId: context.driverUserId, - roomId, - roomKey, - tokenPrefix: params.tokenPrefix, - }); - }); -} - -export async function runMatrixQaE2eeBasicReplyScenario( - context: MatrixQaScenarioContext, -): Promise { - const result = await runMatrixQaE2eeTopLevelScenario(context, { - scenarioId: "matrix-e2ee-basic-reply", - tokenPrefix: "MATRIX_QA_E2EE_BASIC", - }); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - roomKey: result.roomKey, - roomId: result.roomId, - }, - details: [ - `encrypted room key: ${result.roomKey}`, - `encrypted room id: ${result.roomId}`, - `driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("E2EE reply", result.reply), - ].join("\n"), - }; -} - -export async function runMatrixQaE2eeStateAfterMissingEncryptionScenario( - context: MatrixQaScenarioContext, -): Promise { - if (!context.restartGatewayAfterStateMutation) { - throw new Error("Matrix E2EE state_after QA scenario requires hard gateway restart support"); - } - const accountId = context.sutAccountId ?? "sut"; - const configPath = requireMatrixQaGatewayConfigPath(context); - const originalAccountConfig = await readMatrixQaGatewayMatrixAccount({ - accountId, - configPath, - }); - const proxy = await startMatrixQaFaultProxy({ - targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl, - ...context.faultProxyObserver, - rules: [buildSyncStateAfterMissingEncryptionFaultRule(context.sutAccessToken)], - }); - let gatewayPatched = false; - try { - await context.restartGatewayAfterStateMutation( - async () => { - await patchMatrixQaGatewayMatrixAccount({ - accountId, - accountPatch: { - homeserver: proxy.baseUrl, - network: { - dangerouslyAllowPrivateNetwork: true, - }, - }, - configPath, - }); - gatewayPatched = true; - }, - { - timeoutMs: context.timeoutMs, - waitAccountId: accountId, - }, - ); - const result = await runMatrixQaE2eeTopLevelScenario(context, { - scenarioId: "matrix-e2ee-state-after-missing-encryption", - tokenPrefix: "MATRIX_QA_E2EE_STATE_AFTER", - }); - const stateAfterHits = proxy - .hits() - .filter((hit) => hit.ruleId === MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID); - if (stateAfterHits.length > 0) { - throw new Error( - `Matrix E2EE gateway still sent ${MATRIX_QA_SYNC_STATE_AFTER_PARAM}=true on /sync`, - ); - } - return { - artifacts: { - driverEventId: result.driverEventId, - faultProxyBaseUrl: proxy.baseUrl, - reply: result.reply, - roomKey: result.roomKey, - roomId: result.roomId, - stateAfterFaultHitCount: stateAfterHits.length, - stateAfterFaultRuleId: MATRIX_QA_SYNC_STATE_AFTER_FAULT_RULE_ID, - strippedSyncStateAfterParam: true, - }, - details: [ - `encrypted room key: ${result.roomKey}`, - `encrypted room id: ${result.roomId}`, - `driver event: ${result.driverEventId}`, - `fault proxy: ${proxy.baseUrl}`, - `state_after sync opt-in hits: ${stateAfterHits.length}`, - ...buildMatrixReplyDetails("E2EE state_after reply", result.reply), - ].join("\n"), - }; - } finally { - if (gatewayPatched) { - await context - .restartGatewayAfterStateMutation( - async () => { - await replaceMatrixQaGatewayMatrixAccount({ - accountConfig: originalAccountConfig, - accountId, - configPath, - }); - }, - { - timeoutMs: context.timeoutMs, - waitAccountId: accountId, - }, - ) - .catch(() => undefined); - } - await proxy.stop().catch(() => undefined); - } -} - -export async function runMatrixQaE2eeThreadFollowUpScenario( - context: MatrixQaScenarioContext, -): Promise { - const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom( - context, - "matrix-e2ee-thread-follow-up", - ); - const result = await withMatrixQaE2eeDriver( - context, - "matrix-e2ee-thread-follow-up", - async (client) => { - await client.prime(); - const rootEventId = await client.sendTextMessage({ - body: `E2EE thread root ${randomUUID().slice(0, 8)}`, - roomId, - }); - const token = buildMatrixQaToken("MATRIX_QA_E2EE_THREAD"); - const driverEventId = await client.sendTextMessage({ - body: buildMentionPrompt(context.sutUserId, token), - mentionUserIds: [context.sutUserId], - replyToEventId: rootEventId, - roomId, - threadRootEventId: rootEventId, - }); - const matched = await client.waitForRoomEvent({ - predicate: (event) => - isMatrixQaExactMarkerReply(event, { - roomId, - sutUserId: context.sutUserId, - token, - }) && - event.relatesTo?.relType === "m.thread" && - event.relatesTo.eventId === rootEventId, - roomId, - timeoutMs: context.timeoutMs, - }); - const reply = buildMatrixE2eeReplyArtifact(matched.event, token); - assertThreadReplyArtifact(reply, { - expectedRootEventId: rootEventId, - label: "E2EE threaded reply", - }); - return { - driverEventId, - reply, - rootEventId, - token, - }; - }, - ); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - rootEventId: result.rootEventId, - roomKey, - roomId, - }, - details: [ - `encrypted room key: ${roomKey}`, - `encrypted room id: ${roomId}`, - `thread root event: ${result.rootEventId}`, - `mention trigger event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("E2EE threaded reply", result.reply), - ].join("\n"), - }; -} - -export async function runMatrixQaE2eeBootstrapSuccessScenario( - context: MatrixQaScenarioContext, -): Promise { - requireMatrixQaPassword(context, "driver"); - return await withMatrixQaE2eeDriver(context, "matrix-e2ee-bootstrap-success", async (client) => { - const initial = await client.bootstrapOwnDeviceVerification(); - assertMatrixQaBootstrapSucceeded("driver initial", initial); - const result = await client.bootstrapOwnDeviceVerification({ - forceResetCrossSigning: true, - }); - assertMatrixQaBootstrapSucceeded("driver", result); - return { - artifacts: { - backupCreatedVersion: result.verification.backupVersion, - bootstrapActor: "driver", - bootstrapSuccess: true, - currentDeviceId: result.verification.deviceId, - recoveryKeyId: result.verification.recoveryKeyId, - recoveryKeyStored: result.verification.recoveryKeyStored, - }, - details: [ - "driver bootstrap and guarded cross-signing reset succeeded through real Matrix crypto bootstrap", - `device verified: ${result.verification.verified ? "yes" : "no"}`, - `cross-signing verified: ${result.verification.crossSigningVerified ? "yes" : "no"}`, - `signed by owner: ${result.verification.signedByOwner ? "yes" : "no"}`, - `cross-signing published: ${result.crossSigning.published ? "yes" : "no"}`, - `room-key backup version: ${result.verification.backupVersion ?? ""}`, - `recovery key id: ${result.verification.recoveryKeyId ?? ""}`, - ].join("\n"), - }; - }); -} - -export async function runMatrixQaE2eeRecoveryKeyLifecycleScenario( - context: MatrixQaScenarioContext, -): Promise { - const driverPassword = requireMatrixQaPassword(context, "driver"); - return await withMatrixQaE2eeDriver( - context, - "matrix-e2ee-recovery-key-lifecycle", - async (client) => { - const { roomId } = resolveMatrixQaE2eeScenarioGroupRoom( - context, - "matrix-e2ee-recovery-key-lifecycle", - ); - const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ - client, - label: "driver", - }); - const recoveryKey = ready.recoveryKey; - const encodedRecoveryKey = recoveryKey?.encodedPrivateKey?.trim(); - if (!encodedRecoveryKey) { - throw new Error("Matrix E2EE bootstrap did not expose an encoded recovery key"); - } - const seededEventId = await client.sendTextMessage({ - body: `E2EE recovery-key restore seed ${randomUUID().slice(0, 8)}`, - roomId, - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const recoveryDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA Recovery Restore Device", - password: driverPassword, - userId: context.driverUserId, - }); - if (!recoveryDevice.deviceId) { - throw new Error("Matrix E2EE recovery login did not return a secondary device id"); - } - const recoveryClient = await createMatrixQaE2eeScenarioClient({ - accessToken: recoveryDevice.accessToken, - actorId: `driver-recovery-${randomUUID().slice(0, 8)}`, - baseUrl: context.baseUrl, - deviceId: recoveryDevice.deviceId, - observedEvents: context.observedEvents, - outputDir: requireMatrixQaE2eeOutputDir(context), - password: recoveryDevice.password, - scenarioId: "matrix-e2ee-recovery-key-lifecycle", - timeoutMs: context.timeoutMs, - userId: recoveryDevice.userId, - }); - let cleanupRecoveryDevice = true; - try { - const recoveryVerification = await recoveryClient.verifyWithRecoveryKey(encodedRecoveryKey); - if (!recoveryVerification.success) { - throw new Error( - `Matrix E2EE recovery device verification failed: ${recoveryVerification.error ?? "unknown error"}`, - ); - } - const restored = await waitForMatrixQaNonEmptyRoomKeyRestore({ - client: recoveryClient, - recoveryKey: encodedRecoveryKey, - timeoutMs: context.timeoutMs, - }); - const reset = await recoveryClient.resetRoomKeyBackup(); - if (!reset.success) { - throw new Error( - `Matrix E2EE room-key backup reset failed: ${reset.error ?? "unknown error"}`, - ); - } - const resetRecoveryKey = await recoveryClient.getRecoveryKey(); - const resetEncodedRecoveryKey = resetRecoveryKey?.encodedPrivateKey?.trim(); - if (resetEncodedRecoveryKey && resetEncodedRecoveryKey !== encodedRecoveryKey) { - const ownerRecovery = await client.verifyWithRecoveryKey(resetEncodedRecoveryKey); - if (!ownerRecovery.success) { - throw new Error( - `Matrix E2EE owner could not refresh recovery key after backup reset: ${ - ownerRecovery.error ?? "unknown error" - }`, - ); - } - } - await recoveryClient.stop(); - await client.stop().catch(() => undefined); - await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); - cleanupRecoveryDevice = false; - return { - artifacts: { - backupCreatedVersion: reset.createdVersion, - backupReset: reset.success, - backupRestored: restored.success, - bootstrapActor: "driver", - bootstrapSuccess: ready.bootstrap?.success ?? true, - recoveryDeviceId: recoveryDevice.deviceId, - recoveryKeyId: recoveryKey?.keyId ?? null, - recoveryKeyUsable: - recoveryVerification.recoveryKeyAccepted && recoveryVerification.backupUsable, - recoveryKeyStored: true, - recoveryVerified: recoveryVerification.deviceOwnerVerified, - restoreImported: restored.imported, - restoreTotal: restored.total, - seededEventId, - }, - details: [ - "driver recovery lifecycle completed through real Matrix recovery APIs", - `bootstrap backup version: ${ready.verification.backupVersion ?? ""}`, - `seeded encrypted event: ${seededEventId}`, - `recovery device: ${recoveryDevice.deviceId}`, - `recovery key usable: ${recoveryVerification.backupUsable ? "yes" : "no"}`, - `recovery device verified: ${recoveryVerification.deviceOwnerVerified ? "yes" : "no"}`, - `restore imported/total: ${restored.imported}/${restored.total}`, - `restore loaded from secret storage: ${restored.loadedFromSecretStorage ? "yes" : "no"}`, - `reset previous version: ${reset.previousVersion ?? ""}`, - `reset created version: ${reset.createdVersion ?? ""}`, - ].join("\n"), - }; - } finally { - if (cleanupRecoveryDevice) { - await recoveryClient.stop().catch(() => undefined); - await client.stop().catch(() => undefined); - await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); - } - } - }, - ); -} - -export async function runMatrixQaE2eeRecoveryOwnerVerificationRequiredScenario( - context: MatrixQaScenarioContext, -): Promise { - const driverPassword = requireMatrixQaPassword(context, "driver"); - return await withMatrixQaE2eeDriver( - context, - "matrix-e2ee-recovery-owner-verification-required", - async (client) => { - const { roomId } = resolveMatrixQaE2eeScenarioGroupRoom( - context, - "matrix-e2ee-recovery-owner-verification-required", - ); - const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ - client, - label: "driver", - }); - const recoveryKey = ready.recoveryKey; - const encodedRecoveryKey = recoveryKey?.encodedPrivateKey?.trim(); - if (!encodedRecoveryKey) { - throw new Error("Matrix E2EE bootstrap did not expose an encoded recovery key"); - } - const seededEventId = await client.sendTextMessage({ - body: `E2EE recovery owner-verification seed ${randomUUID().slice(0, 8)}`, - roomId, - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const recoveryDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA Owner Verification Required Device", - password: driverPassword, - userId: context.driverUserId, - }); - if (!recoveryDevice.deviceId) { - throw new Error("Matrix E2EE recovery login did not return a secondary device id"); - } - try { - const faulted = await runMatrixQaFaultedRecoveryOwnerVerification({ - accessToken: recoveryDevice.accessToken, - context, - deviceId: recoveryDevice.deviceId, - encodedRecoveryKey, - userId: recoveryDevice.userId, - }); - assertMatrixQaFaultedRecoveryOwnerVerificationRequired(faulted); - return { - artifacts: { - backupRestored: faulted.restore.success, - backupUsable: faulted.verification.backupUsable, - faultHitCount: faulted.faultHits.length, - faultedEndpoints: faulted.faultHits.map((hit) => hit.path), - faultRuleId: MATRIX_QA_OWNER_SIGNATURE_UPLOAD_BLOCKED_RULE_ID, - recoveryDeviceId: recoveryDevice.deviceId, - recoveryKeyAccepted: faulted.verification.recoveryKeyAccepted, - recoveryKeyId: recoveryKey?.keyId ?? null, - recoveryVerified: faulted.verification.deviceOwnerVerified, - restoreImported: faulted.restore.imported, - restoreTotal: faulted.restore.total, - verificationSuccess: faulted.verification.success, - }, - details: [ - "driver recovery key unlocked backup while owner signature upload was blocked", - `seeded encrypted event: ${seededEventId}`, - `recovery device: ${recoveryDevice.deviceId}`, - `fault hits: ${faulted.faultHits.length}`, - `recovery key accepted: ${faulted.verification.recoveryKeyAccepted ? "yes" : "no"}`, - `backup usable: ${faulted.verification.backupUsable ? "yes" : "no"}`, - `device owner verified: ${faulted.verification.deviceOwnerVerified ? "yes" : "no"}`, - `restore imported/total: ${faulted.restore.imported}/${faulted.restore.total}`, - ].join("\n"), - }; - } finally { - await client.stop().catch(() => undefined); - await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); - } - }, - ); -} - -function assertMatrixQaCliE2eeStatus( - label: string, - status: MatrixQaCliVerificationStatus, - opts: { allowUntrustedMatchingKey?: boolean } = {}, -) { - if ( - status.verified !== true || - status.crossSigningVerified !== true || - status.signedByOwner !== true || - !isMatrixQaCliBackupUsable(status.backup, opts) - ) { - throw new Error( - `${label} did not leave the CLI account fully verified and backup-usable: ownerVerified=${ - status.verified === true && - status.crossSigningVerified === true && - status.signedByOwner === true - ? "yes" - : "no" - }, backupUsable=${isMatrixQaCliBackupUsable(status.backup, opts) ? "yes" : "no"}${ - status.backup?.keyLoadError ? `, backupError=${status.backup.keyLoadError}` : "" - }`, - ); - } -} - -function assertMatrixQaCliAccountAddBootstrapStatus(params: { - expectedBackupVersion?: string | null; - expectedUserId: string; - status: MatrixQaCliVerificationStatus; -}) { - const { expectedBackupVersion, expectedUserId, status } = params; - if (status.encryptionEnabled !== true || status.recoveryKeyStored !== true) { - throw new Error( - "Matrix CLI account add --enable-e2ee degraded status did not keep encryption and recovery key state", - ); - } - if (status.userId !== expectedUserId) { - throw new Error( - `Matrix CLI account add --enable-e2ee status user mismatch: expected ${expectedUserId}, got ${status.userId ?? ""}`, - ); - } - if (!status.deviceId || status.serverDeviceKnown !== true) { - throw new Error( - "Matrix CLI account add --enable-e2ee degraded status did not resolve the current server-known device", - ); - } - if (expectedBackupVersion && status.backupVersion !== expectedBackupVersion) { - throw new Error( - `Matrix CLI account add --enable-e2ee backup version mismatch: expected ${expectedBackupVersion}, got ${status.backupVersion ?? ""}`, - ); - } - if (status.backup?.keyLoadError) { - throw new Error( - `Matrix CLI account add --enable-e2ee degraded status reported backup key error: ${status.backup.keyLoadError}`, - ); - } -} - -async function runMatrixQaCliExpectedFailure(params: { - args: string[]; - start: (args: string[], timeoutMs?: number) => MatrixQaCliSession; - timeoutMs: number; -}): Promise { - const session = params.start(params.args, params.timeoutMs); - try { - const result = await session.wait(); - throw new Error( - `${formatMatrixQaCliCommand(params.args)} unexpectedly succeeded with stdout:\n${redactMatrixQaCliOutput( - result.stdout, - )}`, - ); - } catch (error) { - if (error instanceof Error && error.message.includes("unexpectedly succeeded")) { - throw error; - } - const output = session.output(); - if (!output.stdout.trim() && !output.stderr.trim()) { - throw error; - } - return { - args: params.args, - exitCode: 1, - stderr: output.stderr, - stdout: output.stdout, - }; - } finally { - session.kill(); - } -} - -function buildMatrixQaCliE2eeAccountConfig(params: { - accountId: string; - accessToken: string; - baseUrl: string; - deviceId: string; - encryption: boolean; - name: string; - password?: string; - userId: string; -}) { - return { - ...buildMatrixQaPluginActivationConfig(), - channels: { - matrix: { - defaultAccount: params.accountId, - accounts: { - [params.accountId]: { - accessToken: params.accessToken, - deviceId: params.deviceId, - encryption: params.encryption, - homeserver: params.baseUrl, - initialSyncLimit: 1, - name: params.name, - network: { - dangerouslyAllowPrivateNetwork: true, - }, - ...(params.password ? { password: params.password } : {}), - startupVerification: "off", - userId: params.userId, - }, - }, - }, - }, - }; -} - -async function readMatrixQaCliConfig(pathname: string): Promise<{ - channels?: { - matrix?: { - accounts?: Record>; - defaultAccount?: string; - }; - }; -}> { - return JSON.parse(await readFile(pathname, "utf8")) as { - channels?: { - matrix?: { - accounts?: Record>; - defaultAccount?: string; - }; - }; - }; -} - -export async function runMatrixQaE2eeCliAccountAddEnableE2eeScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli-add-e2ee"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Account Add Owner", - scenarioId: "matrix-e2ee-cli-account-add-enable-e2ee", - }); - const cli = await createMatrixQaCliE2eeSetupRuntime({ - artifactLabel: "cli-account-add-enable-e2ee", - context, - }); - try { - const addResult = await cli.run([ - "matrix", - "account", - "add", - "--account", - accountId, - "--name", - "Matrix QA CLI Account Add E2EE", - "--homeserver", - context.baseUrl, - "--user-id", - account.userId, - "--password", - account.password, - "--device-name", - "OpenClaw Matrix QA CLI Account Add E2EE", - "--allow-private-network", - "--enable-e2ee", - "--json", - ]); - const addArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "account-add-enable-e2ee", - result: addResult, - rootDir: cli.rootDir, - }); - const added = parseMatrixQaCliJson(addResult) as MatrixQaCliAccountAddStatus; - if (added.accountId !== accountId || added.encryptionEnabled !== true) { - throw new Error( - "Matrix CLI account add did not report E2EE enabled for the expected account", - ); - } - if (added.verificationBootstrap?.attempted !== true) { - throw new Error("Matrix CLI account add did not attempt verification bootstrap"); - } - if (added.verificationBootstrap.success !== true) { - throw new Error( - `Matrix CLI account add verification bootstrap failed: ${added.verificationBootstrap.error ?? "unknown error"}`, - ); - } - - const statusResult = await cli.run([ - "matrix", - "verify", - "status", - "--account", - accountId, - "--allow-degraded-local-state", - "--json", - ]); - const statusArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "verify-status", - result: statusResult, - rootDir: cli.rootDir, - }); - const status = parseMatrixQaCliJson(statusResult) as MatrixQaCliVerificationStatus; - assertMatrixQaCliAccountAddBootstrapStatus({ - expectedBackupVersion: added.verificationBootstrap.backupVersion, - expectedUserId: account.userId, - status, - }); - const cliDeviceId = status.deviceId ?? null; - - return { - artifacts: { - accountId, - backupVersion: added.verificationBootstrap.backupVersion ?? null, - cliDeviceId, - encryptionEnabled: added.encryptionEnabled, - verificationBootstrapAttempted: added.verificationBootstrap.attempted, - verificationBootstrapSuccess: added.verificationBootstrap.success, - }, - details: [ - "Matrix CLI account add --enable-e2ee created an encrypted account and bootstrapped recovery state", - `account add stdout: ${addArtifacts.stdoutPath}`, - `account add stderr: ${addArtifacts.stderrPath}`, - `verify status stdout: ${statusArtifacts.stdoutPath}`, - `verify status stderr: ${statusArtifacts.stderrPath}`, - `cli device: ${cliDeviceId ?? ""}`, - `cli verified by owner: ${status.verified ? "yes" : "no"}`, - `cli backup usable: ${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}`, - ].join("\n"), - }; - } finally { - await cli.dispose(); - } -} - -export async function runMatrixQaE2eeCliEncryptionSetupScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli-encryption-setup"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Encryption Setup Owner", - scenarioId: "matrix-e2ee-cli-encryption-setup", - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const cliDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA CLI Encryption Setup Device", - password: account.password, - userId: account.userId, - }); - if (!cliDevice.deviceId) { - throw new Error("Matrix E2EE CLI encryption setup login did not return a device id"); - } - const cli = await createMatrixQaCliE2eeSetupRuntime({ - artifactLabel: "cli-encryption-setup", - context, - initialConfig: buildMatrixQaCliE2eeAccountConfig({ - accountId, - accessToken: cliDevice.accessToken, - baseUrl: context.baseUrl, - deviceId: cliDevice.deviceId, - encryption: false, - name: "Matrix QA CLI Encryption Setup", - password: account.password, - userId: cliDevice.userId, - }), - }); - try { - const setupResult = await cli.run([ - "matrix", - "encryption", - "setup", - "--account", - accountId, - "--json", - ]); - const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "encryption-setup", - result: setupResult, - rootDir: cli.rootDir, - }); - const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; - if ( - setup.accountId !== accountId || - setup.success !== true || - setup.encryptionChanged !== true || - setup.bootstrap?.success !== true || - !setup.status - ) { - throw new Error( - `Matrix CLI encryption setup did not report a successful E2EE upgrade: ${setup.bootstrap?.error ?? "unknown error"}`, - ); - } - assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup", setup.status); - - const statusResult = await cli.run([ - "matrix", - "verify", - "status", - "--account", - accountId, - "--json", - ]); - const statusArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "verify-status", - result: statusResult, - rootDir: cli.rootDir, - }); - const status = parseMatrixQaCliJson(statusResult) as MatrixQaCliVerificationStatus; - assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup status", status); - - return { - artifacts: { - accountId, - cliDeviceId: status.deviceId ?? cliDevice.deviceId, - encryptionChanged: setup.encryptionChanged, - setupSuccess: setup.success, - verificationBootstrapSuccess: setup.bootstrap.success, - }, - details: [ - "Matrix CLI encryption setup upgraded an existing account and bootstrapped verification", - `encryption setup stdout: ${setupArtifacts.stdoutPath}`, - `encryption setup stderr: ${setupArtifacts.stderrPath}`, - `verify status stdout: ${statusArtifacts.stdoutPath}`, - `verify status stderr: ${statusArtifacts.stderrPath}`, - `cli device: ${status.deviceId ?? cliDevice.deviceId}`, - `cli verified by owner: ${status.verified ? "yes" : "no"}`, - `cli backup usable: ${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}`, - ].join("\n"), - }; - } finally { - await cli.dispose(); - } -} - -export async function runMatrixQaE2eeCliEncryptionSetupIdempotentScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli-encryption-idempotent"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Encryption Idempotent Owner", - scenarioId: "matrix-e2ee-cli-encryption-setup-idempotent", - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const cliDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA CLI Encryption Idempotent Device", - password: account.password, - userId: account.userId, - }); - if (!cliDevice.deviceId) { - throw new Error("Matrix E2EE CLI idempotent setup login did not return a device id"); - } - const cli = await createMatrixQaCliE2eeSetupRuntime({ - artifactLabel: "cli-encryption-setup-idempotent", - context, - initialConfig: buildMatrixQaCliE2eeAccountConfig({ - accountId, - accessToken: cliDevice.accessToken, - baseUrl: context.baseUrl, - deviceId: cliDevice.deviceId, - encryption: true, - name: "Matrix QA CLI Encryption Setup Idempotent", - password: account.password, - userId: cliDevice.userId, - }), - }); - try { - const setupArgs = ["matrix", "encryption", "setup", "--account", accountId, "--json"]; - const firstResult = await cli.run(setupArgs); - const firstArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "encryption-setup-first", - result: firstResult, - rootDir: cli.rootDir, - }); - const first = parseMatrixQaCliJson(firstResult) as MatrixQaCliEncryptionSetupStatus; - if ( - first.accountId !== accountId || - first.success !== true || - first.encryptionChanged !== false || - first.bootstrap?.success !== true || - !first.status - ) { - throw new Error( - `Matrix CLI encryption setup was not idempotent on first run: ${first.bootstrap?.error ?? "unknown error"}`, - ); - } - assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup idempotent first run", first.status); - - const secondResult = await cli.run(setupArgs); - const secondArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "encryption-setup-second", - result: secondResult, - rootDir: cli.rootDir, - }); - const second = parseMatrixQaCliJson(secondResult) as MatrixQaCliEncryptionSetupStatus; - if ( - second.accountId !== accountId || - second.success !== true || - second.encryptionChanged !== false || - second.bootstrap?.success !== true || - !second.status - ) { - throw new Error( - `Matrix CLI encryption setup was not idempotent on second run: ${second.bootstrap?.error ?? "unknown error"}`, - ); - } - assertMatrixQaCliE2eeStatus("Matrix CLI encryption setup idempotent second run", second.status); - - return { - artifacts: { - accountId, - cliDeviceId: second.status.deviceId ?? cliDevice.deviceId, - firstEncryptionChanged: first.encryptionChanged, - secondEncryptionChanged: second.encryptionChanged, - setupSuccess: second.success, - verificationBootstrapSuccess: second.bootstrap.success, - }, - details: [ - "Matrix CLI encryption setup stayed idempotent on an already encrypted account", - `first setup stdout: ${firstArtifacts.stdoutPath}`, - `first setup stderr: ${firstArtifacts.stderrPath}`, - `second setup stdout: ${secondArtifacts.stdoutPath}`, - `second setup stderr: ${secondArtifacts.stderrPath}`, - `cli device: ${second.status.deviceId ?? cliDevice.deviceId}`, - `first encryption changed: ${first.encryptionChanged ? "yes" : "no"}`, - `second encryption changed: ${second.encryptionChanged ? "yes" : "no"}`, - ].join("\n"), - }; - } finally { - await cli.dispose(); - } -} - -export async function runMatrixQaE2eeCliEncryptionSetupBootstrapFailureScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli-encryption-failure"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Encryption Failure Owner", - scenarioId: "matrix-e2ee-cli-encryption-setup-bootstrap-failure", - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const cliDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA CLI Encryption Failure Device", - password: account.password, - userId: account.userId, - }); - if (!cliDevice.deviceId) { - throw new Error("Matrix E2EE CLI bootstrap-failure login did not return a device id"); - } - const proxy = await startMatrixQaFaultProxy({ - targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl, - ...context.faultProxyObserver, - rules: [buildRoomKeyBackupUnavailableFaultRule(cliDevice.accessToken)], - }); - const cli = await createMatrixQaCliE2eeSetupRuntime({ - artifactLabel: "cli-encryption-setup-bootstrap-failure", - context, - initialConfig: buildMatrixQaCliE2eeAccountConfig({ - accountId, - accessToken: cliDevice.accessToken, - baseUrl: proxy.baseUrl, - deviceId: cliDevice.deviceId, - encryption: false, - name: "Matrix QA CLI Encryption Setup Bootstrap Failure", - password: account.password, - userId: cliDevice.userId, - }), - }); - try { - const failed = await runMatrixQaCliExpectedFailure({ - args: ["matrix", "encryption", "setup", "--account", accountId, "--json"], - start: cli.start, - timeoutMs: context.timeoutMs, - }); - const artifacts = await writeMatrixQaCliOutputArtifacts({ - label: "encryption-setup-bootstrap-failure", - result: failed, - rootDir: cli.rootDir, - }); - const payload = parseMatrixQaCliJson(failed) as MatrixQaCliEncryptionSetupStatus; - if (payload.success !== false && payload.bootstrap?.success !== false) { - throw new Error("Matrix CLI encryption setup failure did not report unsuccessful bootstrap"); - } - const faultHits = proxy.hits(); - if (faultHits.length === 0) { - throw new Error("Matrix CLI encryption setup bootstrap-failure proxy was not exercised"); - } - const bootstrapError = payload.bootstrap?.error ?? ""; - if (!bootstrapError.toLowerCase().includes("room key backup")) { - throw new Error( - `Matrix CLI encryption setup failed for an unexpected reason: ${bootstrapError}`, - ); - } - - return { - artifacts: { - accountId, - bootstrapErrorPreview: truncateUtf16Safe(bootstrapError, 240), - bootstrapSuccess: false, - cliDeviceId: cliDevice.deviceId, - faultedEndpoint: faultHits[0]?.path, - faultHitCount: faultHits.length, - faultRuleId: MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID, - }, - details: [ - "Matrix CLI encryption setup surfaced a bootstrap failure from a faulted room-key backup endpoint", - `failure stdout: ${artifacts.stdoutPath}`, - `failure stderr: ${artifacts.stderrPath}`, - `fault hits: ${faultHits.length}`, - `fault endpoint: ${faultHits[0]?.path ?? ""}`, - `bootstrap error: ${bootstrapError}`, - ].join("\n"), - }; - } finally { - await Promise.all([cli.dispose(), proxy.stop().catch(() => undefined)]); - } -} - -export async function runMatrixQaE2eeCliRecoveryKeySetupScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli-recovery-key-setup"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Recovery Key Owner", - scenarioId: "matrix-e2ee-cli-recovery-key-setup", - }); - const owner = await createMatrixQaE2eeCliOwnerClient({ - account, - context, - scenarioId: "matrix-e2ee-cli-recovery-key-setup", - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ - client: owner, - label: "driver", - }); - const encodedRecoveryKey = ready.recoveryKey?.encodedPrivateKey?.trim(); - if (!encodedRecoveryKey) { - await owner.stop().catch(() => undefined); - throw new Error("Matrix E2EE CLI recovery-key setup did not expose a recovery key"); - } - const cliDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA CLI Recovery Key Setup Device", - password: account.password, - userId: account.userId, - }); - if (!cliDevice.deviceId) { - await owner.stop().catch(() => undefined); - throw new Error("Matrix E2EE CLI recovery-key setup login did not return a device id"); - } - const cli = await createMatrixQaCliE2eeSetupRuntime({ - artifactLabel: "cli-recovery-key-setup", - context, - initialConfig: buildMatrixQaCliE2eeAccountConfig({ - accountId, - accessToken: cliDevice.accessToken, - baseUrl: context.baseUrl, - deviceId: cliDevice.deviceId, - encryption: false, - name: "Matrix QA CLI Recovery Key Setup", - password: account.password, - userId: cliDevice.userId, - }), - }); - try { - const setupResult = await cli.run([ - "matrix", - "encryption", - "setup", - "--account", - accountId, - "--recovery-key", - encodedRecoveryKey, - "--json", - ]); - const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "recovery-key-setup", - result: setupResult, - rootDir: cli.rootDir, - }); - const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; - if ( - setup.accountId !== accountId || - setup.success !== true || - setup.encryptionChanged !== true || - setup.bootstrap?.success !== true || - !setup.status - ) { - throw new Error( - `Matrix CLI recovery-key encryption setup did not succeed: ${setup.bootstrap?.error ?? "unknown error"}`, - ); - } - assertMatrixQaCliE2eeStatus("Matrix CLI recovery-key encryption setup", setup.status, { - allowUntrustedMatchingKey: true, - }); - - return { - artifacts: { - accountId, - backupVersion: setup.status.backupVersion ?? ready.verification.backupVersion ?? null, - cliDeviceId: setup.status.deviceId ?? cliDevice.deviceId, - encryptionChanged: setup.encryptionChanged, - recoveryKeyId: ready.recoveryKey?.keyId ?? null, - recoveryKeyStored: true, - setupSuccess: setup.success, - verificationBootstrapSuccess: setup.bootstrap.success, - }, - details: [ - "Matrix CLI encryption setup accepted a recovery key on a second device", - `recovery setup stdout: ${setupArtifacts.stdoutPath}`, - `recovery setup stderr: ${setupArtifacts.stderrPath}`, - `owner backup version: ${ready.verification.backupVersion ?? ""}`, - `recovery key id: ${ready.recoveryKey?.keyId ?? ""}`, - `cli device: ${setup.status.deviceId ?? cliDevice.deviceId}`, - `cli verified by owner: ${setup.status.verified ? "yes" : "no"}`, - `cli backup usable: ${ - isMatrixQaCliBackupUsable(setup.status.backup, { allowUntrustedMatchingKey: true }) - ? "yes" - : "no" - }`, - ].join("\n"), - }; - } finally { - try { - await owner.stop().catch(() => undefined); - await owner.deleteOwnDevices([cliDevice.deviceId]).catch(() => undefined); - } finally { - await cli.dispose(); - } - } -} - -export async function runMatrixQaE2eeCliRecoveryKeyInvalidScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli-invalid-recovery-key"; - const invalidRecoveryKey = "not-a-valid-matrix-recovery-key"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Invalid Recovery Key Owner", - scenarioId: "matrix-e2ee-cli-recovery-key-invalid", - }); - const owner = await createMatrixQaE2eeCliOwnerClient({ - account, - context, - scenarioId: "matrix-e2ee-cli-recovery-key-invalid", - }); - const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ - client: owner, - label: "cli invalid recovery-key owner", - }); - if (!ready.recoveryKey?.encodedPrivateKey?.trim()) { - await owner.stop().catch(() => undefined); - throw new Error("Matrix E2EE CLI invalid recovery-key setup did not seed secret storage"); - } - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const cliDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA CLI Invalid Recovery Key Device", - password: account.password, - userId: account.userId, - }); - if (!cliDevice.deviceId) { - await owner.stop().catch(() => undefined); - throw new Error("Matrix E2EE CLI invalid recovery-key login did not return a device id"); - } - const cli = await createMatrixQaCliE2eeSetupRuntime({ - artifactLabel: "cli-recovery-key-invalid", - context, - initialConfig: buildMatrixQaCliE2eeAccountConfig({ - accountId, - accessToken: cliDevice.accessToken, - baseUrl: context.baseUrl, - deviceId: cliDevice.deviceId, - encryption: false, - name: "Matrix QA CLI Invalid Recovery Key", - password: account.password, - userId: cliDevice.userId, - }), - }); - try { - const failed = await runMatrixQaCliExpectedFailure({ - args: [ - "matrix", - "encryption", - "setup", - "--account", - accountId, - "--recovery-key", - invalidRecoveryKey, - "--json", - ], - start: cli.start, - timeoutMs: context.timeoutMs, - }); - const artifacts = await writeMatrixQaCliOutputArtifacts({ - label: "recovery-key-invalid", - result: failed, - rootDir: cli.rootDir, - }); - const payload = parseMatrixQaCliJson(failed) as MatrixQaCliEncryptionSetupStatus & { - error?: string; - }; - if (payload.success !== false && payload.bootstrap?.success !== false) { - throw new Error("Matrix CLI invalid recovery-key setup did not report failure"); - } - const failure = payload.bootstrap?.error ?? payload.error ?? ""; - if (!/recovery|secret|key/i.test(failure)) { - throw new Error( - `Matrix CLI invalid recovery-key setup failed for an unexpected reason: ${failure}`, - ); - } - if (failed.stdout.includes(invalidRecoveryKey) || failed.stderr.includes(invalidRecoveryKey)) { - throw new Error("Matrix CLI invalid recovery-key output leaked the recovery key"); - } - - return { - artifacts: { - accountId, - bootstrapErrorPreview: truncateUtf16Safe(failure, 240), - bootstrapSuccess: false, - cliDeviceId: cliDevice.deviceId, - encryptionChanged: payload.encryptionChanged, - recoveryKeyAccepted: false, - recoveryKeyRejected: true, - setupSuccess: false, - }, - details: [ - "Matrix CLI encryption setup rejected an invalid recovery key without leaking it", - `failure stdout: ${artifacts.stdoutPath}`, - `failure stderr: ${artifacts.stderrPath}`, - `cli device: ${cliDevice.deviceId}`, - `failure: ${failure}`, - ].join("\n"), - }; - } finally { - try { - await owner.stop().catch(() => undefined); - await owner.deleteOwnDevices([cliDevice.deviceId]).catch(() => undefined); - } finally { - await cli.dispose(); - } - } -} - -export async function runMatrixQaE2eeCliEncryptionSetupMultiAccountScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli-multi-target"; - const decoyAccountId = "cli-multi-decoy"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Multi Account Owner", - scenarioId: "matrix-e2ee-cli-encryption-setup-multi-account", - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const cliDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA CLI Multi Account Target Device", - password: account.password, - userId: account.userId, - }); - if (!cliDevice.deviceId) { - throw new Error("Matrix E2EE CLI multi-account setup login did not return a device id"); - } - const cli = await createMatrixQaCliE2eeSetupRuntime({ - artifactLabel: "cli-encryption-setup-multi-account", - context, - initialConfig: { - ...buildMatrixQaPluginActivationConfig(), - channels: { - matrix: { - defaultAccount: decoyAccountId, - accounts: { - [decoyAccountId]: { - accessToken: "decoy-token", - deviceId: "DECOYDEVICE", - encryption: false, - homeserver: context.baseUrl, - initialSyncLimit: 1, - name: "Matrix QA CLI Multi Account Decoy", - startupVerification: "off", - userId: "@decoy:matrix-qa.test", - }, - [accountId]: { - accessToken: cliDevice.accessToken, - deviceId: cliDevice.deviceId, - encryption: false, - homeserver: context.baseUrl, - initialSyncLimit: 1, - name: "Matrix QA CLI Multi Account Target", - network: { - dangerouslyAllowPrivateNetwork: true, - }, - password: account.password, - startupVerification: "off", - userId: cliDevice.userId, - }, - }, - }, - }, - }, - }); - try { - const setupResult = await cli.run([ - "matrix", - "encryption", - "setup", - "--account", - accountId, - "--json", - ]); - const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "encryption-setup-multi-account", - result: setupResult, - rootDir: cli.rootDir, - }); - const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; - if ( - setup.accountId !== accountId || - setup.success !== true || - setup.encryptionChanged !== true || - setup.bootstrap?.success !== true || - !setup.status - ) { - throw new Error( - `Matrix CLI multi-account encryption setup did not target the requested account: ${setup.bootstrap?.error ?? "unknown error"}`, - ); - } - assertMatrixQaCliE2eeStatus("Matrix CLI multi-account encryption setup", setup.status); - - const config = await readMatrixQaCliConfig(cli.configPath); - const matrix = config.channels?.matrix; - const target = matrix?.accounts?.[accountId]; - const decoy = matrix?.accounts?.[decoyAccountId]; - const defaultAccountPreserved = matrix?.defaultAccount === decoyAccountId; - const decoyAccountPreserved = - decoy?.encryption === false && - decoy?.accessToken === "decoy-token" && - decoy?.deviceId === "DECOYDEVICE"; - if (!defaultAccountPreserved) { - throw new Error("Matrix CLI multi-account setup changed the default account"); - } - if (!decoyAccountPreserved) { - throw new Error("Matrix CLI multi-account setup mutated the decoy account"); - } - if (target?.encryption !== true) { - throw new Error("Matrix CLI multi-account setup did not enable encryption on the target"); - } - - return { - artifacts: { - accountId, - cliDeviceId: setup.status.deviceId ?? cliDevice.deviceId, - decoyAccountPreserved, - defaultAccountPreserved, - encryptionChanged: setup.encryptionChanged, - setupSuccess: setup.success, - verificationBootstrapSuccess: setup.bootstrap.success, - }, - details: [ - "Matrix CLI encryption setup changed only the requested account in a multi-account config", - `setup stdout: ${setupArtifacts.stdoutPath}`, - `setup stderr: ${setupArtifacts.stderrPath}`, - `default account preserved: ${defaultAccountPreserved ? "yes" : "no"}`, - `decoy account preserved: ${decoyAccountPreserved ? "yes" : "no"}`, - `cli device: ${setup.status.deviceId ?? cliDevice.deviceId}`, - ].join("\n"), - }; - } finally { - await cli.dispose(); - } -} - -export async function runMatrixQaE2eeCliSetupThenGatewayReplyScenario( - context: MatrixQaScenarioContext, -): Promise { - if (!context.restartGatewayAfterStateMutation) { - throw new Error( - "Matrix CLI setup gateway reply scenario requires hard gateway restart support", - ); - } - const gatewayConfigPath = requireMatrixQaGatewayConfigPath(context); - const accountId = "cli-setup-gateway"; - const scenarioId = "matrix-e2ee-cli-setup-then-gateway-reply"; - const roomKey = buildMatrixQaE2eeScenarioRoomKey(scenarioId); - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Setup Gateway", - scenarioId, - }); - const driverAccount = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Setup Driver", - scenarioId, - }); - const driverApi = createMatrixQaClient({ - accessToken: driverAccount.accessToken, - baseUrl: context.baseUrl, - }); - const gatewayApi = createMatrixQaClient({ - accessToken: account.accessToken, - baseUrl: context.baseUrl, - }); - const roomId = await driverApi.createPrivateRoom({ - encrypted: true, - inviteUserIds: [account.userId], - name: "Matrix QA CLI Setup Gateway E2EE", - }); - await gatewayApi.joinRoom(roomId); - - const accountConfig = { - accessToken: account.accessToken, - deviceId: account.deviceId, - dm: { - allowFrom: [driverAccount.userId], - enabled: true, - policy: "allowlist", - sessionScope: "per-room", - threadReplies: "inbound", - }, - enabled: true, - encryption: false, - groupAllowFrom: [driverAccount.userId], - groupPolicy: "allowlist", - groups: { - [roomId]: { - enabled: true, - requireMention: true, - }, - }, - homeserver: context.baseUrl, - initialSyncLimit: 1, - name: "Matrix QA CLI Setup Gateway", - network: { - dangerouslyAllowPrivateNetwork: true, - }, - password: account.password, - startupVerification: "off", - threadReplies: "inbound", - userId: account.userId, - }; - await context.restartGatewayAfterStateMutation( - async () => { - await replaceMatrixQaGatewayMatrixAccount({ - accountConfig, - accountId, - configPath: gatewayConfigPath, - }); - }, - { - timeoutMs: context.timeoutMs, - waitAccountId: accountId, - }, - ); - await context.waitGatewayAccountReady?.(accountId, { - timeoutMs: context.timeoutMs, - }); - const cli = await createMatrixQaCliGatewayRuntime({ - artifactLabel: "cli-setup-then-gateway-reply", - context, - }); - try { - const setupResult = await cli.run([ - "matrix", - "encryption", - "setup", - "--account", - accountId, - "--json", - ]); - const setupArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "encryption-setup", - result: setupResult, - rootDir: cli.rootDir, - }); - const setup = parseMatrixQaCliJson(setupResult) as MatrixQaCliEncryptionSetupStatus; - if ( - setup.accountId !== accountId || - setup.success !== true || - setup.bootstrap?.success !== true - ) { - throw new Error( - `Matrix CLI gateway account setup did not succeed: ${setup.bootstrap?.error ?? "unknown error"}`, - ); - } - if (setup.status) { - assertMatrixQaCliE2eeStatus("Matrix CLI gateway account setup", setup.status); - } - await context.restartGatewayAfterStateMutation( - async () => { - await patchMatrixQaGatewayMatrixAccount({ - accountPatch: { - encryption: true, - password: account.password, - }, - accountId, - configPath: gatewayConfigPath, - }); - }, - { - timeoutMs: context.timeoutMs, - waitAccountId: accountId, - }, - ); - await context.waitGatewayAccountReady?.(accountId, { - timeoutMs: context.timeoutMs, - }); - const driverClient = await createMatrixQaE2eeScenarioClient({ - accessToken: driverAccount.accessToken, - actorId: `driver-cli-setup-gateway-${randomUUID().slice(0, 8)}`, - baseUrl: context.baseUrl, - deviceId: driverAccount.deviceId, - observedEvents: context.observedEvents, - outputDir: requireMatrixQaE2eeOutputDir(context), - password: driverAccount.password, - scenarioId, - timeoutMs: context.timeoutMs, - userId: driverAccount.userId, - }); - const replied = await (async () => { - try { - await ensureMatrixQaE2eeOwnDeviceVerified({ - client: driverClient, - label: "Matrix CLI setup scenario driver", - }); - await driverClient.waitForJoinedMember({ - roomId, - timeoutMs: context.timeoutMs, - userId: account.userId, - }); - await driverClient.prime(); - const token = buildMatrixQaToken("MATRIX_QA_E2EE_CLI_GATEWAY"); - const driverEventId = await driverClient.sendTextMessage({ - body: buildMentionPrompt(account.userId, token), - mentionUserIds: [account.userId], - roomId, - }); - const matched = await driverClient.waitForRoomEvent({ - predicate: (event) => - isMatrixQaExactMarkerReply(event, { - roomId, - sutUserId: account.userId, - token, - }) && event.relatesTo === undefined, - roomId, - timeoutMs: context.timeoutMs, - }); - const reply = buildMatrixE2eeReplyArtifact(matched.event, token); - assertTopLevelReplyArtifact("gateway reply", reply); - return { - driverEventId, - reply, - }; - } finally { - await driverClient.stop(); - } - })(); - - return { - artifacts: { - accountId, - cliDeviceId: setup.status?.deviceId ?? account.deviceId ?? null, - driverUserId: driverAccount.userId, - encryptionChanged: setup.encryptionChanged, - gatewayReply: replied.reply, - gatewayUserId: account.userId, - roomKey, - roomId, - setupSuccess: setup.success, - verificationBootstrapSuccess: setup.bootstrap.success, - }, - details: [ - "Matrix CLI encryption setup left the gateway able to reply in an encrypted room", - `setup stdout: ${setupArtifacts.stdoutPath}`, - `setup stderr: ${setupArtifacts.stderrPath}`, - `driver user: ${driverAccount.userId}`, - `gateway user: ${account.userId}`, - `encrypted room key: ${roomKey}`, - `encrypted room id: ${roomId}`, - `driver event: ${replied.driverEventId}`, - ...buildMatrixReplyDetails("gateway reply", replied.reply), - ].join("\n"), - }; - } finally { - await cli.dispose(); - } -} - -export async function runMatrixQaE2eeCliSelfVerificationScenario( - context: MatrixQaScenarioContext, -): Promise { - const accountId = "cli"; - const account = await registerMatrixQaCliE2eeAccount({ - context, - deviceName: "OpenClaw Matrix QA CLI Self Verification Owner", - scenarioId: "matrix-e2ee-cli-self-verification", - }); - const owner = await createMatrixQaE2eeCliOwnerClient({ - account, - context, - scenarioId: "matrix-e2ee-cli-self-verification", - }); - try { - const ownerReady = await ensureMatrixQaE2eeOwnDeviceVerified({ - client: owner, - label: "CLI self-verification owner", - }); - const encodedRecoveryKey = ownerReady.recoveryKey?.encodedPrivateKey?.trim(); - if (!encodedRecoveryKey) { - throw new Error("Matrix E2EE self-verification scenario did not expose a recovery key"); - } - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const cliDevice = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA CLI Self Verification Device", - password: account.password, - userId: account.userId, - }); - if (!cliDevice.deviceId) { - throw new Error("Matrix E2EE CLI verification login did not return a device id"); - } - - const cli = await createMatrixQaCliSelfVerificationRuntime({ - accountId, - accessToken: cliDevice.accessToken, - context, - deviceId: cliDevice.deviceId, - userId: cliDevice.userId, - }); - try { - const restoreResult = await cli.run( - [ - "matrix", - "verify", - "backup", - "restore", - "--account", - accountId, - "--recovery-key-stdin", - "--json", - ], - context.timeoutMs, - `${encodedRecoveryKey}\n`, - ); - const restoreArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "verify-backup-restore", - result: restoreResult, - rootDir: cli.rootDir, - }); - const restored = parseMatrixQaCliJson(restoreResult) as MatrixQaCliBackupRestoreStatus; - if ( - restored.success !== true || - restored.backup?.decryptionKeyCached !== true || - restored.backup?.matchesDecryptionKey !== true || - restored.backup?.keyLoadError - ) { - throw new Error( - `Matrix CLI recovery key did not load matching room-key backup material before self-verification: ${ - restored.error ?? restored.backup?.keyLoadError ?? "unknown backup state" - }`, - ); - } - const session = cli.start( - [ - "matrix", - "verify", - "self", - "--account", - accountId, - "--timeout-ms", - String(context.timeoutMs), - ], - context.timeoutMs * 2, - ); - try { - const requestOutput = await session.waitForOutput( - (output) => output.text.includes("Accept this verification request"), - "self-verification request guidance", - context.timeoutMs, - ); - const cliTransactionId = parseMatrixQaCliSummaryField(requestOutput.text, "Transaction id"); - const ownerRequested = await waitForMatrixQaVerificationSummary({ - client: owner, - label: "owner received CLI self-verification request", - predicate: (summary) => - isMatrixQaCliOwnerSelfVerification({ - cliDeviceId: cliTransactionId ? undefined : cliDevice.deviceId, - ownerUserId: account.userId, - requirePending: true, - summary, - transactionId: cliTransactionId ?? undefined, - }), - timeoutMs: context.timeoutMs, - }); - if (ownerRequested.canAccept) { - await owner.acceptVerification(ownerRequested.id); - } - - const sasOutput = await session.waitForOutput( - (output) => /^SAS (?:emoji|decimals):/m.test(output.text), - "SAS emoji or decimals", - context.timeoutMs, - ); - const cliSas = parseMatrixQaCliSasText( - sasOutput.text, - "interactive openclaw matrix verify self", - ); - const ownerSas = await waitForMatrixQaVerificationSummary({ - client: owner, - label: "owner SAS for CLI self-verification", - predicate: (summary) => - isMatrixQaCliOwnerSelfVerification({ - cliDeviceId: cliTransactionId ? undefined : cliDevice.deviceId, - ownerUserId: account.userId, - requireSas: true, - summary, - transactionId: cliTransactionId ?? undefined, - }), - timeoutMs: context.timeoutMs, - }); - const sasArtifact = assertMatrixQaCliSasMatches({ - cliSas, - owner: ownerSas, - }); - const ownerConfirm = owner.confirmVerificationSas(ownerSas.id); - await session.writeStdin("yes\n"); - session.endStdin(); - await ownerConfirm; - const completedCli = await session.wait(); - const selfVerificationArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "verify-self", - result: completedCli, - rootDir: cli.rootDir, - }); - if (!/^Device verified by owner:\s*yes$/m.test(completedCli.stdout)) { - throw new Error( - "Interactive Matrix CLI self-verification did not report final device verification", - ); - } - if (!/^Cross-signing verified:\s*yes$/m.test(completedCli.stdout)) { - throw new Error( - "Interactive Matrix CLI self-verification did not report full Matrix identity trust", - ); - } - const completedOwner = await waitForMatrixQaVerificationSummary({ - client: owner, - label: "owner completed CLI self-verification", - predicate: (summary) => - isMatrixQaCliOwnerSelfVerification({ - cliDeviceId: cliTransactionId ? undefined : cliDevice.deviceId, - ownerUserId: account.userId, - requireCompleted: true, - summary, - transactionId: cliTransactionId ?? undefined, - }), - timeoutMs: context.timeoutMs, - }); - const cliVerificationId = - completedCli.stdout.match(/^Verification id:\s*(\S+)/m)?.[1] ?? "interactive-cli"; - const statusResult = await cli.run([ - "matrix", - "verify", - "status", - "--account", - accountId, - "--json", - ]); - const statusArtifacts = await writeMatrixQaCliOutputArtifacts({ - label: "verify-status", - result: statusResult, - rootDir: cli.rootDir, - }); - const status = parseMatrixQaCliJson(statusResult) as MatrixQaCliVerificationStatus; - if ( - status.verified !== true || - status.crossSigningVerified !== true || - status.signedByOwner !== true || - status.backup?.trusted !== true || - status.backup?.matchesDecryptionKey !== true || - status.backup?.keyLoadError - ) { - throw new Error( - `Matrix CLI device was not fully usable after SAS completion: ownerVerified=${ - status.verified === true && - status.crossSigningVerified === true && - status.signedByOwner === true - ? "yes" - : "no" - }, backupUsable=${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}${ - status.backup?.keyLoadError ? `, backupError=${status.backup.keyLoadError}` : "" - }`, - ); - } - return { - artifacts: { - completedVerificationIds: [cliVerificationId, completedOwner.id], - currentDeviceId: status.deviceId ?? cliDevice.deviceId, - ...(cliSas.kind === "emoji" ? { sasEmoji: sasArtifact } : {}), - secondaryDeviceId: cliDevice.deviceId, - }, - details: [ - "Matrix CLI self-verification established full Matrix identity trust through interactive openclaw matrix verify self", - "cli secret config cleaned after run: yes", - `cli backup restore stdout: ${restoreArtifacts.stdoutPath}`, - `cli backup restore stderr: ${restoreArtifacts.stderrPath}`, - `cli verify self stdout: ${selfVerificationArtifacts.stdoutPath}`, - `cli verify self stderr: ${selfVerificationArtifacts.stderrPath}`, - `cli verify status stdout: ${statusArtifacts.stdoutPath}`, - `cli verify status stderr: ${statusArtifacts.stderrPath}`, - `cli device: ${cliDevice.deviceId}`, - `cli verification id: ${cliVerificationId}`, - `owner-side verification id: ${completedOwner.id}`, - `transaction: ${completedOwner.transactionId ?? ""}`, - `cli verified by owner: ${status.verified ? "yes" : "no"}`, - `cli cross-signing verified: ${status.crossSigningVerified ? "yes" : "no"}`, - `cli backup usable: ${isMatrixQaCliBackupUsable(status.backup) ? "yes" : "no"}`, - ].join("\n"), - }; - } finally { - session.kill(); - } - } finally { - try { - await cli.dispose(); - } finally { - await owner.stop().catch(() => undefined); - await owner.deleteOwnDevices([cliDevice.deviceId]).catch(() => undefined); - } - } - } finally { - await owner.stop().catch(() => undefined); - } -} - -export async function runMatrixQaE2eeDeviceSasVerificationScenario( - context: MatrixQaScenarioContext, -): Promise { - requireMatrixQaPassword(context, "driver"); - requireMatrixQaPassword(context, "observer"); - if (!context.observerDeviceId) { - throw new Error("Matrix E2EE observer device id is required for device SAS verification"); - } - if (!context.driverDeviceId) { - throw new Error("Matrix E2EE driver device id is required for device SAS verification"); - } - const observerDeviceId = context.observerDeviceId; - const driverDeviceId = context.driverDeviceId; - return await withMatrixQaE2eeDriverAndObserver( - context, - "matrix-e2ee-device-sas-verification", - async ({ driver, observer }) => { - await Promise.all([ - ensureMatrixQaE2eeOwnDeviceVerified({ - client: driver, - label: "driver", - }), - ensureMatrixQaE2eeOwnDeviceVerified({ - client: observer, - label: "observer", - }), - ]); - const result = await completeMatrixQaSasVerification({ - initiator: driver, - recipient: observer, - recipientUserId: context.observerUserId, - request: { - deviceId: observerDeviceId, - userId: context.observerUserId, - }, - timeoutMs: context.timeoutMs, - }); - const driverTrust = await assertMatrixQaPeerDeviceTrusted({ - client: driver, - deviceId: observerDeviceId, - label: "driver", - timeoutMs: context.timeoutMs, - userId: context.observerUserId, - }); - const observerTrust = await assertMatrixQaPeerDeviceTrusted({ - client: observer, - deviceId: driverDeviceId, - label: "observer", - timeoutMs: context.timeoutMs, - userId: context.driverUserId, - }); - return { - artifacts: { - completedVerificationIds: [result.completedInitiator.id, result.completedRecipient.id], - currentDeviceId: driverDeviceId, - driverTrustsObserverDevice: driverTrust.verified, - observerTrustsDriverDevice: observerTrust.verified, - sasEmoji: result.sasEmoji, - secondaryDeviceId: observerDeviceId, - }, - details: [ - "driver-to-observer device verification completed with real SAS", - `initiator transaction: ${result.completedInitiator.transactionId ?? ""}`, - `recipient transaction: ${result.completedRecipient.transactionId ?? ""}`, - `driver trusts observer device: ${driverTrust.verified ? "yes" : "no"}`, - `observer trusts driver device: ${observerTrust.verified ? "yes" : "no"}`, - `emoji: ${result.sasEmoji.join(", ")}`, - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeQrVerificationScenario( - context: MatrixQaScenarioContext, -): Promise { - requireMatrixQaPassword(context, "driver"); - requireMatrixQaPassword(context, "observer"); - if (!context.observerDeviceId) { - throw new Error("Matrix E2EE observer device id is required for QR verification"); - } - if (!context.driverDeviceId) { - throw new Error("Matrix E2EE driver device id is required for QR verification"); - } - const observerDeviceId = context.observerDeviceId; - const driverDeviceId = context.driverDeviceId; - return await withMatrixQaE2eeDriverAndObserver( - context, - "matrix-e2ee-qr-verification", - async ({ driver, observer }) => { - await Promise.all([ - ensureMatrixQaE2eeOwnDeviceVerified({ - client: driver, - label: "driver", - }), - ensureMatrixQaE2eeOwnDeviceVerified({ - client: observer, - label: "observer", - }), - ]); - const initiated = await driver.requestVerification({ - deviceId: observerDeviceId, - userId: context.observerUserId, - }); - const incoming = await waitForMatrixQaVerificationSummary({ - client: observer, - label: "QR recipient request", - predicate: (summary) => - !summary.initiatedByMe && sameMatrixQaVerificationTransaction(summary, initiated), - timeoutMs: context.timeoutMs, - }); - if (incoming.canAccept) { - await observer.acceptVerification(incoming.id); - } - await waitForMatrixQaVerificationSummary({ - client: driver, - label: "QR request ready", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, initiated) && summary.phaseName === "ready", - timeoutMs: context.timeoutMs, - }); - const qr = await driver.generateVerificationQr(initiated.id); - await observer.scanVerificationQr(incoming.id, qr.qrDataBase64); - const reciprocate = await waitForMatrixQaVerificationSummary({ - client: driver, - label: "QR reciprocate", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, initiated) && summary.hasReciprocateQr, - timeoutMs: context.timeoutMs, - }); - await driver.confirmVerificationReciprocateQr(reciprocate.id); - const qrByteCount = Buffer.from(qr.qrDataBase64, "base64").byteLength; - const completedDriver = await waitForMatrixQaVerificationSummary({ - client: driver, - label: "QR driver complete", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, initiated) && summary.completed, - timeoutMs: context.timeoutMs, - }); - const completedObserver = await waitForMatrixQaVerificationSummary({ - client: observer, - label: "QR observer complete", - predicate: (summary) => - sameMatrixQaVerificationTransaction(summary, completedDriver) && summary.completed, - timeoutMs: context.timeoutMs, - }); - const driverTrust = await assertMatrixQaPeerDeviceTrusted({ - client: driver, - deviceId: observerDeviceId, - label: "driver", - timeoutMs: context.timeoutMs, - userId: context.observerUserId, - }); - const observerTrust = await assertMatrixQaPeerDeviceTrusted({ - client: observer, - deviceId: driverDeviceId, - label: "observer", - timeoutMs: context.timeoutMs, - userId: context.driverUserId, - }); - return { - artifacts: { - completedVerificationIds: [completedDriver.id, completedObserver.id], - driverTrustsObserverDevice: driverTrust.verified, - identityVerificationCompleted: true, - observerTrustsDriverDevice: observerTrust.verified, - qrBytes: qrByteCount, - secondaryDeviceId: observerDeviceId, - }, - details: [ - "driver-to-observer QR verification completed through real QR scan", - `transaction: ${completedDriver.transactionId ?? ""}`, - `driver trusts observer device: ${driverTrust.verified ? "yes" : "no"}`, - `observer trusts driver device: ${observerTrust.verified ? "yes" : "no"}`, - `qr bytes: ${qrByteCount}`, - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeStaleDeviceHygieneScenario( - context: MatrixQaScenarioContext, -): Promise { - const driverPassword = requireMatrixQaPassword(context, "driver"); - return await withMatrixQaE2eeDriver( - context, - "matrix-e2ee-stale-device-hygiene", - async (client) => { - await ensureMatrixQaE2eeOwnDeviceVerified({ - client, - label: "driver", - }); - const loginClient = createMatrixQaClient({ - baseUrl: context.baseUrl, - }); - const secondary = await loginClient.loginWithPassword({ - deviceName: "OpenClaw Matrix QA Stale Device", - password: driverPassword, - userId: context.driverUserId, - }); - if (!secondary.deviceId) { - throw new Error("Matrix stale-device login did not return a secondary device id"); - } - const before = await client.listOwnDevices(); - if (!before.some((device) => device.deviceId === secondary.deviceId)) { - throw new Error("Matrix stale-device list did not include the secondary login"); - } - await client.stop().catch(() => undefined); - const deleted = await client.deleteOwnDevices([secondary.deviceId]); - const remainingDeviceIds = deleted.remainingDevices.map((device) => device.deviceId); - if (remainingDeviceIds.includes(secondary.deviceId)) { - throw new Error( - "Matrix stale-device deletion left the secondary device in the device list", - ); - } - if ( - deleted.currentDeviceId && - !deleted.remainingDevices.some((device) => device.deviceId === deleted.currentDeviceId) - ) { - throw new Error("Matrix stale-device deletion removed the current device"); - } - return { - artifacts: { - currentDeviceId: deleted.currentDeviceId, - deletedDeviceIds: deleted.deletedDeviceIds, - remainingDeviceIds, - secondaryDeviceId: secondary.deviceId, - }, - details: [ - "driver secondary device was created, observed, and removed through real device APIs", - `current device: ${deleted.currentDeviceId ?? ""}`, - `deleted device: ${secondary.deviceId}`, - `remaining devices: ${remainingDeviceIds.join(", ")}`, - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeDmSasVerificationScenario( - context: MatrixQaScenarioContext, -): Promise { - requireMatrixQaPassword(context, "driver"); - requireMatrixQaPassword(context, "observer"); - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY); - return await withMatrixQaE2eeDriverAndObserver( - context, - "matrix-e2ee-dm-sas-verification", - async ({ driver, observer }) => { - await Promise.all([ - ensureMatrixQaE2eeOwnDeviceVerified({ - client: driver, - label: "driver", - }), - ensureMatrixQaE2eeOwnDeviceVerified({ - client: observer, - label: "observer", - }), - ]); - const result = await completeMatrixQaSasVerification({ - initiator: driver, - recipient: observer, - recipientUserId: context.observerUserId, - request: { - roomId, - userId: context.observerUserId, - }, - timeoutMs: context.timeoutMs, - }); - if ( - result.completedInitiator.roomId !== roomId || - result.completedRecipient.roomId !== roomId - ) { - throw new Error("Matrix E2EE DM verification completed outside the expected DM room"); - } - return { - artifacts: { - completedVerificationIds: [result.completedInitiator.id, result.completedRecipient.id], - roomKey: MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY, - sasEmoji: result.sasEmoji, - verificationRoomId: roomId, - }, - details: [ - "driver/observer encrypted DM verification completed with SAS in the expected room", - `verification DM room: ${roomId}`, - `transaction: ${result.completedInitiator.transactionId ?? ""}`, - `emoji: ${result.sasEmoji.join(", ")}`, - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeRestartResumeScenario( - context: MatrixQaScenarioContext, -): Promise { - if (!context.restartGateway) { - throw new Error("Matrix E2EE restart scenario requires gateway restart support"); - } - const restartGateway = context.restartGateway; - return await withMatrixQaIsolatedE2eeDriverRoom( - context, - "matrix-e2ee-restart-resume", - async ({ client, driverUserId, roomId, roomKey }) => { - const first = await runMatrixQaE2eeTopLevelWithClient(context, { - client, - driverUserId, - roomId, - roomKey, - tokenPrefix: "MATRIX_QA_E2EE_BEFORE_RESTART", - }); - await restartGateway(); - const recovered = await runMatrixQaE2eeTopLevelWithClient(context, { - client, - driverUserId, - roomId, - roomKey, - tokenPrefix: "MATRIX_QA_E2EE_AFTER_RESTART", - }); - return { - artifacts: { - driverUserId, - firstDriverEventId: first.driverEventId, - firstReply: first.reply, - recoveredDriverEventId: recovered.driverEventId, - recoveredReply: recovered.reply, - restartSignal: "gateway-restart", - roomKey: recovered.roomKey, - roomId: recovered.roomId, - }, - details: [ - `encrypted room key: ${recovered.roomKey}`, - `encrypted room id: ${recovered.roomId}`, - `isolated driver user: ${driverUserId}`, - `pre-restart event: ${first.driverEventId}`, - ...buildMatrixReplyDetails("pre-restart reply", first.reply), - `post-restart event: ${recovered.driverEventId}`, - ...buildMatrixReplyDetails("post-restart reply", recovered.reply), - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeVerificationNoticeNoTriggerScenario( - context: MatrixQaScenarioContext, -): Promise { - const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom( - context, - "matrix-e2ee-verification-notice-no-trigger", - ); - return await withMatrixQaE2eeDriver( - context, - "matrix-e2ee-verification-notice-no-trigger", - async (client) => { - await client.prime(); - const token = buildMatrixQaToken("MATRIX_QA_E2EE_VERIFY_NOTICE"); - const body = `Matrix verification started with ${context.driverUserId}; ${buildMentionPrompt( - context.sutUserId, - token, - )}`; - const noticeSentAt = Date.now(); - const noticeEventId = await client.sendNoticeMessage({ - body, - mentionUserIds: [context.sutUserId], - roomId, - }); - const result = await client.waitForOptionalRoomEvent({ - predicate: (event) => - isMatrixQaE2eeNoticeTriggeredSutReply({ - event, - noticeEventId, - noticeSentAt, - roomId, - sutUserId: context.sutUserId, - token, - }), - roomId, - timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs), - }); - if (result.matched) { - throw new Error(`unexpected E2EE verification-notice reply: ${result.event.eventId}`); - } - return { - artifacts: { - expectedNoReplyWindowMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs), - noticeEventId, - roomKey, - roomId, - }, - details: [ - `encrypted room key: ${roomKey}`, - `encrypted room id: ${roomId}`, - `verification notice event: ${noticeEventId}`, - `waited ${resolveMatrixQaNoReplyWindowMs(context.timeoutMs)}ms with no SUT reply`, - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeArtifactRedactionScenario( - context: MatrixQaScenarioContext, -): Promise { - return await withMatrixQaIsolatedE2eeDriverRoom( - context, - "matrix-e2ee-artifact-redaction", - async ({ client, driverUserId, roomId, roomKey }) => { - const result = await runMatrixQaE2eeTopLevelWithClient(context, { - client, - driverUserId, - roomId, - roomKey, - tokenPrefix: "MATRIX_QA_E2EE_REDACT", - }); - const leaked = context.observedEvents.some( - (event) => - event.roomId === result.roomId && - (event.body?.includes(result.token) || event.formattedBody?.includes(result.token)), - ); - if (!leaked) { - throw new Error( - "Matrix E2EE redaction scenario did not observe decrypted content in memory", - ); - } - return { - artifacts: { - driverEventId: result.driverEventId, - driverUserId, - reply: result.reply, - roomKey: result.roomKey, - roomId: result.roomId, - }, - details: [ - "decrypted E2EE payload reached in-memory assertions only", - "observed-event artifacts redact body/formatted_body unless OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1", - `encrypted room id: ${result.roomId}`, - `isolated driver user: ${driverUserId}`, - ...buildMatrixReplyDetails("E2EE reply", result.reply), - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeMediaImageScenario( - context: MatrixQaScenarioContext, -): Promise { - return await withMatrixQaIsolatedE2eeDriverRoom( - context, - "matrix-e2ee-media-image", - async ({ client, driverUserId, roomId, roomKey }) => { - const startSince = await client.prime(); - const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId); - const driverEventId = await client.sendImageMessage({ - body: triggerBody, - buffer: createMatrixQaSplitColorImagePng(), - contentType: "image/png", - fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, - mentionUserIds: [context.sutUserId], - roomId, - }); - const attachmentEvent = await client.waitForRoomEvent({ - predicate: (event) => - event.roomId === roomId && - event.eventId === driverEventId && - event.sender === driverUserId && - event.attachment?.kind === "image" && - event.attachment.caption === triggerBody, - roomId, - timeoutMs: context.timeoutMs, - }); - const matched = await client.waitForRoomEvent({ - predicate: (event) => - event.roomId === roomId && - event.sender === context.sutUserId && - event.type === "m.room.message" && - event.relatesTo === undefined && - hasMatrixQaExpectedColorReply(event.body), - roomId, - timeoutMs: context.timeoutMs, - }); - const reply: MatrixQaReplyArtifact = { - eventId: matched.event.eventId, - mentions: matched.event.mentions, - relatesTo: matched.event.relatesTo, - sender: matched.event.sender, - }; - return { - artifacts: { - attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, - driverEventId, - driverUserId, - reply, - roomKey, - roomId, - }, - details: [ - `encrypted room key: ${roomKey}`, - `encrypted room id: ${roomId}`, - `isolated driver user: ${driverUserId}`, - `driver encrypted image event: ${driverEventId}`, - `driver encrypted image filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`, - `driver encrypted image since: ${attachmentEvent.since ?? startSince ?? ""}`, - ...buildMatrixReplyDetails("E2EE image reply", reply), - ].join("\n"), - }; - }, - ); -} - -export async function runMatrixQaE2eeKeyBootstrapFailureScenario( - context: MatrixQaScenarioContext, -): Promise { - const { faultHits, result } = await runMatrixQaFaultedE2eeBootstrap(context); - const bootstrapError = assertMatrixQaExpectedBootstrapFailure({ faultHits, result }); - - return { - artifacts: { - bootstrapActor: "driver", - bootstrapErrorPreview: truncateUtf16Safe(bootstrapError, 240), - bootstrapSuccess: result.success, - faultedEndpoint: MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT, - faultHitCount: faultHits.length, - ...(faultHits[0]?.ruleId ? { faultRuleId: faultHits[0].ruleId } : {}), - }, - details: [ - "Matrix E2EE bootstrap failure surfaced through real SDK bootstrap.", - `faulted endpoint: GET ${MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT}`, - `fault hits: ${faultHits.length}`, - `bootstrap success: ${result.success ? "yes" : "no"}`, - `bootstrap error: ${bootstrapError || ""}`, - ].join("\n"), - }; -} -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index ccaf61662dcb..fdccd58a9abb 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -21,6 +21,21 @@ import { runApprovalPluginMetadataSingleEventScenario, runApprovalThreadTargetScenario, } from "./scenario-runtime-approval.js"; +import { + runMatrixQaE2eeCliAccountAddEnableE2eeScenario, + runMatrixQaE2eeCliEncryptionSetupBootstrapFailureScenario, + runMatrixQaE2eeCliEncryptionSetupIdempotentScenario, + runMatrixQaE2eeCliEncryptionSetupScenario, +} from "./scenario-runtime-e2ee-cli-account.js"; +import { + runMatrixQaE2eeCliEncryptionSetupMultiAccountScenario, + runMatrixQaE2eeCliSetupThenGatewayReplyScenario, +} from "./scenario-runtime-e2ee-cli-gateway.js"; +import { + runMatrixQaE2eeCliRecoveryKeyInvalidScenario, + runMatrixQaE2eeCliRecoveryKeySetupScenario, +} from "./scenario-runtime-e2ee-cli-recovery.js"; +import { runMatrixQaE2eeCliSelfVerificationScenario } from "./scenario-runtime-e2ee-cli-verification.js"; import { runMatrixQaE2eeCorruptCryptoIdbSnapshotScenario, runMatrixQaE2eeHistoryExistsBackupEmptyScenario, @@ -38,29 +53,24 @@ import { import { runMatrixQaE2eeArtifactRedactionScenario, runMatrixQaE2eeBasicReplyScenario, - runMatrixQaE2eeBootstrapSuccessScenario, - runMatrixQaE2eeCliAccountAddEnableE2eeScenario, - runMatrixQaE2eeCliEncryptionSetupBootstrapFailureScenario, - runMatrixQaE2eeCliEncryptionSetupIdempotentScenario, - runMatrixQaE2eeCliEncryptionSetupMultiAccountScenario, - runMatrixQaE2eeCliEncryptionSetupScenario, - runMatrixQaE2eeCliRecoveryKeyInvalidScenario, - runMatrixQaE2eeCliRecoveryKeySetupScenario, - runMatrixQaE2eeCliSetupThenGatewayReplyScenario, - runMatrixQaE2eeCliSelfVerificationScenario, - runMatrixQaE2eeDeviceSasVerificationScenario, - runMatrixQaE2eeDmSasVerificationScenario, - runMatrixQaE2eeKeyBootstrapFailureScenario, runMatrixQaE2eeMediaImageScenario, - runMatrixQaE2eeQrVerificationScenario, - runMatrixQaE2eeRecoveryKeyLifecycleScenario, - runMatrixQaE2eeRecoveryOwnerVerificationRequiredScenario, runMatrixQaE2eeRestartResumeScenario, runMatrixQaE2eeStateAfterMissingEncryptionScenario, - runMatrixQaE2eeStaleDeviceHygieneScenario, runMatrixQaE2eeThreadFollowUpScenario, runMatrixQaE2eeVerificationNoticeNoTriggerScenario, -} from "./scenario-runtime-e2ee.js"; +} from "./scenario-runtime-e2ee-messages.js"; +import { + runMatrixQaE2eeBootstrapSuccessScenario, + runMatrixQaE2eeKeyBootstrapFailureScenario, + runMatrixQaE2eeRecoveryKeyLifecycleScenario, + runMatrixQaE2eeRecoveryOwnerVerificationRequiredScenario, +} from "./scenario-runtime-e2ee-recovery.js"; +import { + runMatrixQaE2eeDeviceSasVerificationScenario, + runMatrixQaE2eeDmSasVerificationScenario, + runMatrixQaE2eeQrVerificationScenario, + runMatrixQaE2eeStaleDeviceHygieneScenario, +} from "./scenario-runtime-e2ee-verification.js"; import { runInboundEditIgnoredScenario, runInboundEditNoDuplicateTriggerScenario, diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 393371cc4bc1..eb5725218f9d 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -5656,7 +5656,7 @@ describe("matrix live qa scenarios", () => { userId: "@cli-recovery:matrix-qa.test", }); let initialAccountConfig: Record | null = null; - runMatrixQaOpenClawCli.mockImplementation(async ({ args, env }) => { + runMatrixQaOpenClawCli.mockImplementation(async ({ args, env, stdin }) => { if (!initialAccountConfig && env.OPENCLAW_CONFIG_PATH) { const initialConfig = JSON.parse( await readFile(String(env.OPENCLAW_CONFIG_PATH), "utf8"), @@ -5673,8 +5673,9 @@ describe("matrix live qa scenarios", () => { const joined = args.join(" "); if ( joined === - "matrix encryption setup --account cli-recovery-key-setup --recovery-key encoded-recovery-key --json" + "matrix encryption setup --account cli-recovery-key-setup --recovery-key-stdin --json" ) { + expect(stdin).toBe("encoded-recovery-key\n"); return { args, exitCode: 0, @@ -5759,8 +5760,7 @@ describe("matrix live qa scenarios", () => { "setup", "--account", "cli-recovery-key-setup", - "--recovery-key", - "encoded-recovery-key", + "--recovery-key-stdin", "--json", ], ]); @@ -5839,6 +5839,8 @@ describe("matrix live qa scenarios", () => { .fn() .mockRejectedValue(new Error("openclaw matrix encryption setup exited 1")); const kill = vi.fn(); + const endStdin = vi.fn(); + const writeStdin = vi.fn().mockResolvedValue(undefined); startMatrixQaOpenClawCli.mockReturnValue({ args: [ "matrix", @@ -5846,15 +5848,15 @@ describe("matrix live qa scenarios", () => { "setup", "--account", "cli-invalid-recovery-key", - "--recovery-key", - "not-a-valid-matrix-recovery-key", + "--recovery-key-stdin", "--json", ], + endStdin, kill, output, wait, waitForOutput: vi.fn(), - writeStdin: vi.fn(), + writeStdin, }); const scenario = requireMatrixQaScenario("matrix-e2ee-cli-recovery-key-invalid"); @@ -5895,10 +5897,11 @@ describe("matrix live qa scenarios", () => { "setup", "--account", "cli-invalid-recovery-key", - "--recovery-key", - "not-a-valid-matrix-recovery-key", + "--recovery-key-stdin", "--json", ]); + expect(writeStdin).toHaveBeenCalledWith("not-a-valid-matrix-recovery-key\n"); + expect(endStdin).toHaveBeenCalledTimes(1); expect(output).toHaveBeenCalledTimes(1); expect(wait).toHaveBeenCalledTimes(1); expect(kill).toHaveBeenCalledTimes(1);