Files
cc-haha/runtime/test_helpers.py
程序员阿江(Relakkes) 5cd6b5d07b Prevent desktop Computer Use from stalling on missing approvals and unstable text input
Desktop sessions were missing a visible request_access approval path and could
mis-detect their own app window as an unapproved frontmost target, which caused
Computer Use clicks to fail even after opening the intended app. On macOS, text
entry was also split across inconsistent clipboard and keystroke paths, making
Electron inputs unreliable for Chinese and short strings.

This change adds a desktop approval bridge over the existing session websocket,
renders a dedicated desktop approval modal, threads the real desktop bundle id
into the Computer Use executor, and switches macOS clipboard typing onto the
native pasteboard plus system paste shortcut path. It also makes tool error
results expandable in the desktop chat UI so frontmost-gate failures are fully
visible during debugging.

Constraint: Desktop sessions run the CLI over the SDK websocket path, so Ink tool JSX dialogs are not visible there
Constraint: macOS IME and Electron text inputs are unreliable with pyautogui.write and generic hotkey synthesis
Rejected: Reuse CLI setToolJSX dialogs in desktop mode | no transport for mid-call Ink UI over the SDK bridge
Rejected: Keep shell pbcopy/pbpaste for clipboard typing | inconsistent with NSPasteboard path and less reliable for Chinese text
Confidence: medium
Scope-risk: moderate
Reversibility: clean
Directive: Keep desktop Computer Use approvals and macOS text-entry behavior on a single bridge/path; avoid reintroducing separate CLI-only and desktop-only codepaths for the same action
Tested: python3 -m unittest runtime/test_helpers.py
Tested: bun test src/utils/computerUse/permissions.test.ts src/server/__tests__/conversation-service.test.ts
Tested: cd desktop && bun run test ComputerUsePermissionModal chatStore
Tested: cd desktop && bun run test chatBlocks
Tested: cd desktop && bun run lint
Not-tested: End-to-end manual Computer Use interaction against a live Electron target app on macOS
2026-04-20 12:12:02 +08:00

323 lines
13 KiB
Python

#!/usr/bin/env python3
"""Cross-platform tests for mac_helper.py and win_helper.py.
Tests the platform-independent parts (JSON protocol, key mapping, capture logic)
without requiring platform-specific dependencies. Can run on any OS with pytest.
Usage:
python -m pytest runtime/test_helpers.py -v
# or simply:
python runtime/test_helpers.py
"""
from __future__ import annotations
import json
import subprocess
import sys
import unittest
from pathlib import Path
from unittest.mock import patch, MagicMock
# Determine which helper to test based on current platform
IS_WINDOWS = sys.platform == "win32"
IS_MACOS = sys.platform == "darwin"
RUNTIME_DIR = Path(__file__).parent
MAC_HELPER = RUNTIME_DIR / "mac_helper.py"
WIN_HELPER = RUNTIME_DIR / "win_helper.py"
class TestKeyMap(unittest.TestCase):
"""Test the KEY_MAP and normalize_key function — platform-independent logic."""
def _load_key_map(self, helper_path: Path) -> dict[str, str]:
"""Extract KEY_MAP from a helper by importing it with mocked deps."""
# Read the file and extract just the KEY_MAP dict
source = helper_path.read_text()
# Find KEY_MAP definition
start = source.index("KEY_MAP = {")
# Find the matching closing brace
depth = 0
for i, ch in enumerate(source[start:], start):
if ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
end = i + 1
break
key_map_source = source[start:end]
ns: dict = {}
exec(key_map_source, ns)
return ns["KEY_MAP"]
def test_mac_key_map_exists(self):
if not MAC_HELPER.exists():
self.skipTest("mac_helper.py not found")
km = self._load_key_map(MAC_HELPER)
self.assertIn("cmd", km)
self.assertIn("ctrl", km)
self.assertEqual(km["cmd"], "command")
self.assertEqual(km["alt"], "option")
def test_win_key_map_exists(self):
if not WIN_HELPER.exists():
self.skipTest("win_helper.py not found")
km = self._load_key_map(WIN_HELPER)
self.assertIn("cmd", km)
self.assertIn("ctrl", km)
# Windows maps cmd/command/meta to 'win' key
self.assertEqual(km["cmd"], "win")
self.assertEqual(km["command"], "win")
self.assertEqual(km["meta"], "win")
# Windows maps alt/option to 'alt'
self.assertEqual(km["alt"], "alt")
self.assertEqual(km["option"], "alt")
def test_common_keys_present_in_both(self):
"""Both helpers must have the same set of key names."""
if not MAC_HELPER.exists() or not WIN_HELPER.exists():
self.skipTest("Both helpers required")
mac_km = self._load_key_map(MAC_HELPER)
win_km = self._load_key_map(WIN_HELPER)
# All keys in mac should be in win and vice versa
self.assertEqual(set(mac_km.keys()), set(win_km.keys()),
"KEY_MAP keys must be identical across platforms")
def test_all_alphabet_keys(self):
"""All a-z keys should map to themselves."""
for helper in [MAC_HELPER, WIN_HELPER]:
if not helper.exists():
continue
km = self._load_key_map(helper)
for char in "abcdefghijklmnopqrstuvwxyz":
self.assertEqual(km[char], char, f"{helper.name}: {char} should map to itself")
def test_all_digit_keys(self):
"""All 0-9 keys should map to themselves."""
for helper in [MAC_HELPER, WIN_HELPER]:
if not helper.exists():
continue
km = self._load_key_map(helper)
for digit in "0123456789":
self.assertEqual(km[digit], digit, f"{helper.name}: {digit} should map to itself")
def test_function_keys(self):
"""F1-F12 should map to themselves."""
for helper in [MAC_HELPER, WIN_HELPER]:
if not helper.exists():
continue
km = self._load_key_map(helper)
for i in range(1, 13):
key = f"f{i}"
self.assertEqual(km[key], key, f"{helper.name}: {key} should map to itself")
class TestJSONProtocol(unittest.TestCase):
"""Test that both helpers follow the same JSON command protocol."""
def _get_helper(self) -> Path:
"""Get the appropriate helper for the current platform."""
if IS_WINDOWS and WIN_HELPER.exists():
return WIN_HELPER
if IS_MACOS and MAC_HELPER.exists():
return MAC_HELPER
return MAC_HELPER if MAC_HELPER.exists() else WIN_HELPER
def _parse_main_commands(self, helper_path: Path) -> list[str]:
"""Extract all command names from the main() dispatcher."""
source = helper_path.read_text()
commands = []
for line in source.splitlines():
stripped = line.strip()
if stripped.startswith('if command == "'):
cmd = stripped.split('"')[1]
commands.append(cmd)
return commands
def test_both_helpers_same_commands(self):
"""Both helpers must support the exact same set of commands."""
if not MAC_HELPER.exists() or not WIN_HELPER.exists():
self.skipTest("Both helpers required")
mac_cmds = set(self._parse_main_commands(MAC_HELPER))
win_cmds = set(self._parse_main_commands(WIN_HELPER))
self.assertEqual(mac_cmds, win_cmds,
f"Command sets differ.\nOnly in mac: {mac_cmds - win_cmds}\nOnly in win: {win_cmds - mac_cmds}")
def test_expected_commands_exist(self):
"""Core commands should be present in each helper."""
expected = {
"check_permissions", "list_displays", "get_display_size",
"screenshot", "resolve_prepare_capture", "zoom",
"prepare_for_action", "preview_hide_set", "find_window_displays",
"key", "hold_key", "type", "click", "drag",
"move_mouse", "scroll", "mouse_down", "mouse_up",
"cursor_position", "frontmost_app", "app_under_point",
"list_installed_apps", "list_running_apps", "open_app",
"read_clipboard", "write_clipboard", "paste_clipboard",
}
for helper in [MAC_HELPER, WIN_HELPER]:
if not helper.exists():
continue
cmds = set(self._parse_main_commands(helper))
missing = expected - cmds
self.assertFalse(missing,
f"{helper.name} missing commands: {missing}")
def test_unknown_command_returns_error(self):
"""Running a non-existent command should return a JSON error."""
helper = self._get_helper()
if not helper.exists():
self.skipTest("No helper found")
# On macOS without venv, mac_helper.py may fail at import (AppKit);
# on Windows without venv, win_helper.py may fail at import (win32gui).
# Only test if the helper can actually import.
check = subprocess.run(
[sys.executable, "-c", f"import importlib.util; "
f"spec = importlib.util.spec_from_file_location('h', '{helper}')"],
capture_output=True, text=True
)
result = subprocess.run(
[sys.executable, str(helper), "nonexistent_command_xyz"],
capture_output=True, text=True
)
if result.returncode == 1 and not result.stdout.strip():
# Import failed — platform deps missing, skip this test
self.skipTest(f"Cannot run {helper.name} on this platform (missing deps)")
# Should exit with code 2
self.assertEqual(result.returncode, 2)
parsed = json.loads(result.stdout.strip())
self.assertFalse(parsed["ok"])
self.assertEqual(parsed["error"]["code"], "bad_command")
class TestHelperOutputFormat(unittest.TestCase):
"""Test the JSON output helpers are consistent."""
def test_json_output_function_exists(self):
"""Both helpers should define json_output and error_output."""
for helper in [MAC_HELPER, WIN_HELPER]:
if not helper.exists():
continue
source = helper.read_text()
self.assertIn("def json_output(", source,
f"{helper.name} missing json_output function")
self.assertIn("def error_output(", source,
f"{helper.name} missing error_output function")
def test_main_entry_point(self):
"""Both helpers should have the standard main entry point."""
for helper in [MAC_HELPER, WIN_HELPER]:
if not helper.exists():
continue
source = helper.read_text()
self.assertIn('if __name__ == "__main__":', source,
f"{helper.name} missing __main__ guard")
self.assertIn("def main()", source,
f"{helper.name} missing main() function")
class TestWinHelperPermissions(unittest.TestCase):
"""Windows-specific: permissions should always return True."""
def test_check_permissions_always_granted(self):
"""On Windows, permissions are not needed — should always be True."""
if not WIN_HELPER.exists():
self.skipTest("win_helper.py not found")
# Extract and exec just the check_permissions function
source = WIN_HELPER.read_text()
# Find the function
self.assertIn("def check_permissions()", source)
# The function should return both as True
# We can verify by reading the source
start = source.index("def check_permissions()")
# Find next def or end
rest = source[start:]
lines = rest.split("\n")
func_lines = [lines[0]]
for line in lines[1:]:
if line and not line[0].isspace() and not line.startswith("#"):
break
func_lines.append(line)
func_source = "\n".join(func_lines)
self.assertIn('"accessibility": True', func_source)
self.assertIn('"screenRecording": True', func_source)
class TestMacHelperPermissions(unittest.TestCase):
"""macOS helper permission detection should use the official trust API."""
def test_check_permissions_uses_ax_api_instead_of_system_events(self):
if not MAC_HELPER.exists():
self.skipTest("mac_helper.py not found")
source = MAC_HELPER.read_text()
self.assertIn("def detect_accessibility_permission()", source)
self.assertIn("AXIsProcessTrusted", source)
start = source.index("def check_permissions()")
rest = source[start:]
lines = rest.split("\n")
func_lines = [lines[0]]
for line in lines[1:]:
if line and not line[0].isspace() and not line.startswith("#"):
break
func_lines.append(line)
func_source = "\n".join(func_lines)
self.assertIn("detect_accessibility_permission()", func_source)
self.assertNotIn('tell application "System Events"', func_source)
def test_clipboard_shortcuts_use_osascript_path(self):
if not MAC_HELPER.exists():
self.skipTest("mac_helper.py not found")
source = MAC_HELPER.read_text()
self.assertIn("def paste_clipboard()", source)
self.assertIn('send_keystroke_via_osascript("v", ["command"])', source)
self.assertIn('if parts == ["command", "v"]:', source)
self.assertIn('elif parts == ["command", "a"]:', source)
class TestCrossPlatformFunctions(unittest.TestCase):
"""Test functions that are identical between both helpers."""
def _get_function_body(self, helper_path: Path, func_name: str) -> str:
"""Extract a function's body (code lines only, no comments/blanks)."""
source = helper_path.read_text()
marker = f"def {func_name}("
if marker not in source:
return ""
start = source.index(marker)
rest = source[start:]
lines = rest.split("\n")
func_lines = [lines[0]]
for line in lines[1:]:
# Stop at next top-level def/class or non-indented non-empty line
stripped = line.strip()
if line and not line[0].isspace() and stripped and not stripped.startswith("#"):
break
# Skip comments and blank lines for comparison
if stripped.startswith("#") or not stripped:
continue
func_lines.append(line)
return " ".join(" ".join(func_lines).split())
def test_input_functions_identical(self):
"""Input action functions (click, scroll, etc.) should be identical."""
if not MAC_HELPER.exists() or not WIN_HELPER.exists():
self.skipTest("Both helpers required")
for func in ["click", "scroll", "hold_keys", "type_text"]:
mac_src = self._get_function_body(MAC_HELPER, func)
win_src = self._get_function_body(WIN_HELPER, func)
self.assertEqual(mac_src, win_src,
f"{func} should be identical across platforms")
if __name__ == "__main__":
unittest.main()