package tools import ( "context" "fmt" "strings" "git.gopher.ink/mirrors2/AutoZero/llm" ) // computer 工具:本地桌面控制(借鉴 Claude Code / ccb 的 Computer Use)。 // 平台实现由 guiBackend 抽象、编译期选择,各自独立成文件: // - computer_darwin.go macOS:osascript(System Events) + screencapture, // 检测到 cliclick 时点击更可靠(保持独立的实现); // - computer_linux.go Linux:xdotool + scrot/import/gnome-screenshot(X11,Wayland 不支持); // - computer_windows.go Windows:PowerShell(SendKeys 键盘 + user32 P/Invoke 鼠标/滚轮)。 // // 分发与参数校验在本文件(共享);校验失败必须发生在任何真实 GUI 调用之前。 // guiBackend 是单平台的桌面控制实现。 type guiBackend interface { Available() error // 依赖检查(缺工具时给出可操作的报错) Screenshot(ctx context.Context, region string) (string, error) // region 为 "x,y,w,h" 或 "" Click(ctx context.Context, x, y int, kind string) error // kind: left / right / double TypeText(ctx context.Context, text string) error KeyChord(ctx context.Context, chordStr string) error Scroll(ctx context.Context, direction string, amount int) error PreflightPermissions() []string // 首次运行引导授予桌面权限;返回缺失项(空=齐全) } // PreflightDesktopPermissions 探测并(首次运行)引导授予桌面控制所需的系统权限 // (macOS 的辅助功能 / 屏幕录制)。返回缺失权限名(空切片=齐全)。非 macOS 由 backend // 直接返回 nil。在程序启动时调用一次,缺失时自动打开系统设置面板并打印指引。 func PreflightDesktopPermissions() []string { return currentGuiBackend.PreflightPermissions() } // currentGuiBackend 由各平台的 newGuiBackend() 在编译期决定。 var currentGuiBackend = newGuiBackend() // Computer 返回 computer 工具(三平台通用入口)。 func Computer() *Tool { return &Tool{ Name: "computer", Description: "Control the local desktop: screenshot (returned as an image you can see), click, double-click, right-click, type text, press key chords, scroll. " + "macOS: needs Accessibility & Screen Recording permission. If a click fails with an accessibility/osascript error (-25211, -1728), STOP retrying clicks and switch to `osascript` (AppleScript/System Events) or the `browser` tool instead. " + "Screenshots are expensive (vision inference ~10-70s each): take one only when you must SEE pixels (e.g. final confirmation for the user); for verification prefer programmatic state reads (browser `js`, `osascript` reading menu/window state). " + "Click coordinates are in real logical screen resolution — do not trust coordinates derived from a screenshot's displayed size. " + "Linux: needs xdotool + scrot/import under X11 (Wayland unsupported). " + "Windows: uses PowerShell (Win key unsupported by SendKeys).", SideEffect: true, Kind: KindExec, // 控制本机桌面:执行类 Parameters: map[string]any{ "type": "object", "properties": map[string]any{ "action": map[string]any{"type": "string", "enum": []string{"screenshot", "left_click", "right_click", "double_click", "type", "key", "scroll"}, "description": "GUI action to perform"}, "x": map[string]any{"type": "integer", "description": "Screen x coordinate (click actions)"}, "y": map[string]any{"type": "integer", "description": "Screen y coordinate (click actions)"}, "text": map[string]any{"type": "string", "description": "Text to type (action=type)"}, "key": map[string]any{"type": "string", "description": "Key or chord, e.g. enter, cmd+c / ctrl+c, cmd+shift+t (action=key)"}, "direction": map[string]any{"type": "string", "enum": []string{"up", "down"}, "description": "Scroll direction (action=scroll, default down)"}, "amount": map[string]any{"type": "integer", "description": "Scroll repetitions 1-10 (action=scroll, default 3)"}, }, "required": []string{"action"}, }, Run: func(ctx context.Context, input string) (Result, error) { m, err := arg(input) if err != nil { return Result{}, err } b := currentGuiBackend if err := b.Available(); err != nil { return Result{}, err } vision := func(path string, err error) (string, error) { if err != nil { return "", err } p, serr := ShrinkForVision(path) if serr != nil { return path, nil } return p, nil } action := str(m, "action") switch action { case "screenshot": path, err := vision(b.Screenshot(ctx, "")) if err != nil { return Result{}, err } return Result{Text: "screenshot saved", Images: []llm.Image{{Path: path}}}, nil case "left_click", "right_click", "double_click": x, y := intArg(m, "x"), intArg(m, "y") if x < 0 || y < 0 { return Result{}, fmt.Errorf("action %q requires valid x,y coordinates", action) } kind := strings.TrimSuffix(action, "_click") if err := b.Click(ctx, x, y, kind); err != nil { return Result{}, err } // 操作后截图验证;截图失败降级为纯文本,不让点击白做。 path, serr := vision(b.Screenshot(ctx, "")) if serr != nil { return TextResult(action + " at " + itoa(x) + "," + itoa(y) + " (post-action screenshot failed: " + serr.Error() + ")"), nil } return Result{Text: action + " at " + itoa(x) + "," + itoa(y), Images: []llm.Image{{Path: path}}}, nil case "type": text := str(m, "text") if text == "" { return Result{}, fmt.Errorf("action type requires non-empty text") } if err := b.TypeText(ctx, text); err != nil { return Result{}, err } return TextResult("typed " + itoa(len([]rune(text))) + " chars"), nil case "key": chordStr := str(m, "key") if _, _, err := parseChord(chordStr); err != nil { return Result{}, err } if err := b.KeyChord(ctx, chordStr); err != nil { return Result{}, err } return TextResult("pressed " + chordStr), nil case "scroll": dir := str(m, "direction") if dir == "" { dir = "down" } n := intArg(m, "amount") if n <= 0 { n = 3 } if n > 10 { n = 10 } if err := b.Scroll(ctx, dir, n); err != nil { return Result{}, err } path, serr := vision(b.Screenshot(ctx, "")) if serr != nil { return TextResult("scrolled " + dir + " (post-action screenshot failed: " + serr.Error() + ")"), nil } return Result{Text: "scrolled " + dir, Images: []llm.Image{{Path: path}}}, nil default: return Result{}, fmt.Errorf("unknown action %q", action) } }, } } // ---- 共享键序模型:统一词表,各平台后端再翻译为自己的语法 ---- var chordMods = map[string]bool{ "cmd": true, "command": true, "ctrl": true, "control": true, "alt": true, "option": true, "shift": true, "fn": true, "win": true, "super": true, "meta": true, } var namedKeys = map[string]bool{ "enter": true, "return": true, "tab": true, "escape": true, "esc": true, "delete": true, "backspace": true, "forwarddelete": true, "up": true, "down": true, "left": true, "right": true, "space": true, "home": true, "end": true, "pageup": true, "pagedown": true, "f1": true, "f2": true, "f3": true, "f4": true, "f5": true, "f6": true, "f7": true, "f8": true, "f9": true, "f10": true, "f11": true, "f12": true, } // fnKeyNum 返回功能键的编号(f1..f12),非功能键返回 0。 func fnKeyNum(main string) int { if len(main) >= 2 && main[0] == 'f' { n := 0 for _, r := range main[1:] { if r < '0' || r > '9' { return 0 } n = n*10 + int(r-'0') } if n >= 1 && n <= 12 { return n } } return 0 } // parseChord 把 "cmd+shift+t" 解析为修饰键 + 主键(命名键或单字符)。 // 全平台共用词表;平台不支持的键由后端在运行时报错。 func parseChord(s string) (mods []string, main string, err error) { if strings.TrimSpace(s) == "" { return nil, "", fmt.Errorf("empty key chord") } for _, part := range strings.Split(strings.ToLower(strings.TrimSpace(s)), "+") { part = strings.TrimSpace(part) if part == "" { return nil, "", fmt.Errorf("empty key in chord %q", s) } if chordMods[part] && main == "" { mods = append(mods, part) continue } if main != "" { return nil, "", fmt.Errorf("multiple main keys in chord %q", s) } if namedKeys[part] || len([]rune(part)) == 1 { main = part continue } return nil, "", fmt.Errorf("unknown key %q", part) } if main == "" { return nil, "", fmt.Errorf("chord %q has no main key", s) } return mods, main, nil } // chordToXdotool 翻译为 xdotool 的 "mod+keysym" 语法(如 "ctrl+c"、"super+q")。 func chordToXdotool(mods []string, main, orig string) (string, error) { modMap := map[string]string{ "ctrl": "ctrl", "control": "ctrl", "alt": "alt", "option": "alt", "shift": "shift", "cmd": "super", "command": "super", "win": "super", "super": "super", "meta": "super", } mainMap := map[string]string{ "enter": "Return", "return": "Return", "escape": "Escape", "esc": "Escape", "tab": "Tab", "delete": "BackSpace", "backspace": "BackSpace", "forwarddelete": "Delete", "up": "Up", "down": "Down", "left": "Left", "right": "Right", "space": "space", "home": "Home", "end": "End", "pageup": "Page_Up", "pagedown": "Page_Down", } parts := make([]string, 0, len(mods)+1) for _, m := range mods { if m == "fn" { return "", fmt.Errorf("fn is not supported on Linux (chord %q)", orig) } parts = append(parts, modMap[m]) } if sym, ok := mainMap[main]; ok { parts = append(parts, sym) } else if n := fnKeyNum(main); n > 0 { parts = append(parts, "F"+itoa(n)) } else { parts = append(parts, main) // 单字符即 keysym } return strings.Join(parts, "+"), nil } // chordToSendKeys 翻译为 Windows SendKeys 语法(^=ctrl %=alt +=shift)。 func chordToSendKeys(mods []string, main, orig string) (string, error) { modMap := map[string]string{"ctrl": "^", "control": "^", "alt": "%", "option": "%", "shift": "+"} prefix := "" for _, m := range mods { if sym, ok := modMap[m]; ok { prefix += sym continue } return "", fmt.Errorf("modifier %q is not supported by Windows SendKeys (chord %q)", m, orig) } mainMap := map[string]string{ "enter": "{ENTER}", "return": "{ENTER}", "tab": "{TAB}", "escape": "{ESC}", "esc": "{ESC}", "delete": "{BS}", "backspace": "{BS}", "forwarddelete": "{DEL}", "up": "{UP}", "down": "{DOWN}", "left": "{LEFT}", "right": "{RIGHT}", "space": " ", "home": "{HOME}", "end": "{END}", "pageup": "{PGUP}", "pagedown": "{PGDN}", } if sym, ok := mainMap[main]; ok { return prefix + sym, nil } if n := fnKeyNum(main); n > 0 { return prefix + "{F" + itoa(n) + "}", nil } c := []rune(main)[0] if strings.ContainsRune("+^%~(){}[]", c) { return prefix + "{" + main + "}", nil } return prefix + main, nil } // sendKeysEscape 把任意文本转义为 SendKeys 字面量(特殊字符包 {}, // 换行/制表转 Enter/Tab)。 func sendKeysEscape(text string) string { var b strings.Builder for _, r := range text { switch r { case '\n': b.WriteString("{ENTER}") case '\t': b.WriteString("{TAB}") case '+', '^', '%', '~', '(', ')', '{', '}', '[', ']': b.WriteString("{" + string(r) + "}") default: b.WriteRune(r) } } return b.String() } // parseRegion 解析 "x,y,w,h"。 func parseRegion(region string) (x, y, w, h int, err error) { if _, err := fmt.Sscanf(strings.ReplaceAll(region, " ", ""), "%d,%d,%d,%d", &x, &y, &w, &h); err != nil { return 0, 0, 0, 0, fmt.Errorf("invalid region %q (want x,y,w,h)", region) } return x, y, w, h, nil } func intArg(m map[string]any, key string) int { if v, ok := m[key].(float64); ok { return int(v) } return -1 } func itoa(n int) string { return fmt.Sprint(n) } // oneLine 把多行命令输出压成单行(用于错误消息)。 func oneLine(b []byte) string { s := string(b) for i, r := range s { if r == '\n' || r == '\r' { s = s[:i] break } } return s }