Sakurasan
d5dac5a6fc
llm（三协议 Provider）/ session（JSONL 持久化）/ tools（Registry、围栏、 builtin、computer）/ cdp（browser）/ skills / mcp / agent（循环、两级压缩、 编排、审批门控）+ 底座 jsonl/textx/execx/diag/scratch/spill/imagecache。 产品装配代码（config 桥接、schedule 工具、memory/plan/retro/prompt） 留在 AutoGo，宿主接入方式见 MIGRATION.md。
332 lines
12 KiB
Go
332 lines
12 KiB
Go
package tools
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
|
||
"git.gopher.ink/mirrors2/AutoZero/llm"
|
||
)
|
||
|
||
// computer 工具：本地桌面控制（借鉴 Claude Code / ccb 的 Computer Use）。
|
||
// 平台实现由 guiBackend 抽象、编译期选择，各自独立成文件：
|
||
// - computer_darwin.go macOS：osascript(System Events) + screencapture，
|
||
// 检测到 cliclick 时点击更可靠（保持独立的实现）；
|
||
// - computer_linux.go Linux：xdotool + scrot/import/gnome-screenshot（X11，Wayland 不支持）；
|
||
// - computer_windows.go Windows：PowerShell（SendKeys 键盘 + user32 P/Invoke 鼠标/滚轮）。
|
||
//
|
||
// 分发与参数校验在本文件（共享）；校验失败必须发生在任何真实 GUI 调用之前。
|
||
|
||
// guiBackend 是单平台的桌面控制实现。
|
||
type guiBackend interface {
|
||
Available() error // 依赖检查（缺工具时给出可操作的报错）
|
||
Screenshot(ctx context.Context, region string) (string, error) // region 为 "x,y,w,h" 或 ""
|
||
Click(ctx context.Context, x, y int, kind string) error // kind: left / right / double
|
||
TypeText(ctx context.Context, text string) error
|
||
KeyChord(ctx context.Context, chordStr string) error
|
||
Scroll(ctx context.Context, direction string, amount int) error
|
||
PreflightPermissions() []string // 首次运行引导授予桌面权限；返回缺失项（空=齐全）
|
||
}
|
||
|
||
// PreflightDesktopPermissions 探测并（首次运行）引导授予桌面控制所需的系统权限
|
||
// （macOS 的辅助功能 / 屏幕录制）。返回缺失权限名（空切片=齐全）。非 macOS 由 backend
|
||
// 直接返回 nil。在程序启动时调用一次，缺失时自动打开系统设置面板并打印指引。
|
||
func PreflightDesktopPermissions() []string {
|
||
return currentGuiBackend.PreflightPermissions()
|
||
}
|
||
|
||
// currentGuiBackend 由各平台的 newGuiBackend() 在编译期决定。
|
||
var currentGuiBackend = newGuiBackend()
|
||
|
||
// Computer 返回 computer 工具（三平台通用入口）。
|
||
func Computer() *Tool {
|
||
return &Tool{
|
||
Name: "computer",
|
||
Description: "Control the local desktop: screenshot (returned as an image you can see), click, double-click, right-click, type text, press key chords, scroll. " +
|
||
"macOS: needs Accessibility & Screen Recording permission. If a click fails with an accessibility/osascript error (-25211, -1728), STOP retrying clicks and switch to `osascript` (AppleScript/System Events) or the `browser` tool instead. " +
|
||
"Screenshots are expensive (vision inference ~10-70s each): take one only when you must SEE pixels (e.g. final confirmation for the user); for verification prefer programmatic state reads (browser `js`, `osascript` reading menu/window state). " +
|
||
"Click coordinates are in real logical screen resolution — do not trust coordinates derived from a screenshot's displayed size. " +
|
||
"Linux: needs xdotool + scrot/import under X11 (Wayland unsupported). " +
|
||
"Windows: uses PowerShell (Win key unsupported by SendKeys).",
|
||
SideEffect: true,
|
||
Kind: KindExec, // 控制本机桌面：执行类
|
||
Parameters: map[string]any{
|
||
"type": "object",
|
||
"properties": map[string]any{
|
||
"action": map[string]any{"type": "string", "enum": []string{"screenshot", "left_click", "right_click", "double_click", "type", "key", "scroll"},
|
||
"description": "GUI action to perform"},
|
||
"x": map[string]any{"type": "integer", "description": "Screen x coordinate (click actions)"},
|
||
"y": map[string]any{"type": "integer", "description": "Screen y coordinate (click actions)"},
|
||
"text": map[string]any{"type": "string", "description": "Text to type (action=type)"},
|
||
"key": map[string]any{"type": "string", "description": "Key or chord, e.g. enter, cmd+c / ctrl+c, cmd+shift+t (action=key)"},
|
||
"direction": map[string]any{"type": "string", "enum": []string{"up", "down"},
|
||
"description": "Scroll direction (action=scroll, default down)"},
|
||
"amount": map[string]any{"type": "integer", "description": "Scroll repetitions 1-10 (action=scroll, default 3)"},
|
||
},
|
||
"required": []string{"action"},
|
||
},
|
||
Run: func(ctx context.Context, input string) (Result, error) {
|
||
m, err := arg(input)
|
||
if err != nil {
|
||
return Result{}, err
|
||
}
|
||
b := currentGuiBackend
|
||
if err := b.Available(); err != nil {
|
||
return Result{}, err
|
||
}
|
||
vision := func(path string, err error) (string, error) {
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
p, serr := ShrinkForVision(path)
|
||
if serr != nil {
|
||
return path, nil
|
||
}
|
||
return p, nil
|
||
}
|
||
action := str(m, "action")
|
||
switch action {
|
||
case "screenshot":
|
||
path, err := vision(b.Screenshot(ctx, ""))
|
||
if err != nil {
|
||
return Result{}, err
|
||
}
|
||
return Result{Text: "screenshot saved", Images: []llm.Image{{Path: path}}}, nil
|
||
|
||
case "left_click", "right_click", "double_click":
|
||
x, y := intArg(m, "x"), intArg(m, "y")
|
||
if x < 0 || y < 0 {
|
||
return Result{}, fmt.Errorf("action %q requires valid x,y coordinates", action)
|
||
}
|
||
kind := strings.TrimSuffix(action, "_click")
|
||
if err := b.Click(ctx, x, y, kind); err != nil {
|
||
return Result{}, err
|
||
}
|
||
// 操作后截图验证；截图失败降级为纯文本，不让点击白做。
|
||
path, serr := vision(b.Screenshot(ctx, ""))
|
||
if serr != nil {
|
||
return TextResult(action + " at " + itoa(x) + "," + itoa(y) + " (post-action screenshot failed: " + serr.Error() + ")"), nil
|
||
}
|
||
return Result{Text: action + " at " + itoa(x) + "," + itoa(y), Images: []llm.Image{{Path: path}}}, nil
|
||
|
||
case "type":
|
||
text := str(m, "text")
|
||
if text == "" {
|
||
return Result{}, fmt.Errorf("action type requires non-empty text")
|
||
}
|
||
if err := b.TypeText(ctx, text); err != nil {
|
||
return Result{}, err
|
||
}
|
||
return TextResult("typed " + itoa(len([]rune(text))) + " chars"), nil
|
||
|
||
case "key":
|
||
chordStr := str(m, "key")
|
||
if _, _, err := parseChord(chordStr); err != nil {
|
||
return Result{}, err
|
||
}
|
||
if err := b.KeyChord(ctx, chordStr); err != nil {
|
||
return Result{}, err
|
||
}
|
||
return TextResult("pressed " + chordStr), nil
|
||
|
||
case "scroll":
|
||
dir := str(m, "direction")
|
||
if dir == "" {
|
||
dir = "down"
|
||
}
|
||
n := intArg(m, "amount")
|
||
if n <= 0 {
|
||
n = 3
|
||
}
|
||
if n > 10 {
|
||
n = 10
|
||
}
|
||
if err := b.Scroll(ctx, dir, n); err != nil {
|
||
return Result{}, err
|
||
}
|
||
path, serr := vision(b.Screenshot(ctx, ""))
|
||
if serr != nil {
|
||
return TextResult("scrolled " + dir + " (post-action screenshot failed: " + serr.Error() + ")"), nil
|
||
}
|
||
return Result{Text: "scrolled " + dir, Images: []llm.Image{{Path: path}}}, nil
|
||
|
||
default:
|
||
return Result{}, fmt.Errorf("unknown action %q", action)
|
||
}
|
||
},
|
||
}
|
||
}
|
||
|
||
// ---- 共享键序模型：统一词表，各平台后端再翻译为自己的语法 ----
|
||
|
||
var chordMods = map[string]bool{
|
||
"cmd": true, "command": true, "ctrl": true, "control": true, "alt": true,
|
||
"option": true, "shift": true, "fn": true, "win": true, "super": true, "meta": true,
|
||
}
|
||
|
||
var namedKeys = map[string]bool{
|
||
"enter": true, "return": true, "tab": true, "escape": true, "esc": true,
|
||
"delete": true, "backspace": true, "forwarddelete": true,
|
||
"up": true, "down": true, "left": true, "right": true,
|
||
"space": true, "home": true, "end": true, "pageup": true, "pagedown": true,
|
||
"f1": true, "f2": true, "f3": true, "f4": true, "f5": true, "f6": true,
|
||
"f7": true, "f8": true, "f9": true, "f10": true, "f11": true, "f12": true,
|
||
}
|
||
|
||
// fnKeyNum 返回功能键的编号（f1..f12），非功能键返回 0。
|
||
func fnKeyNum(main string) int {
|
||
if len(main) >= 2 && main[0] == 'f' {
|
||
n := 0
|
||
for _, r := range main[1:] {
|
||
if r < '0' || r > '9' {
|
||
return 0
|
||
}
|
||
n = n*10 + int(r-'0')
|
||
}
|
||
if n >= 1 && n <= 12 {
|
||
return n
|
||
}
|
||
}
|
||
return 0
|
||
}
|
||
|
||
// parseChord 把 "cmd+shift+t" 解析为修饰键 + 主键（命名键或单字符）。
|
||
// 全平台共用词表；平台不支持的键由后端在运行时报错。
|
||
func parseChord(s string) (mods []string, main string, err error) {
|
||
if strings.TrimSpace(s) == "" {
|
||
return nil, "", fmt.Errorf("empty key chord")
|
||
}
|
||
for _, part := range strings.Split(strings.ToLower(strings.TrimSpace(s)), "+") {
|
||
part = strings.TrimSpace(part)
|
||
if part == "" {
|
||
return nil, "", fmt.Errorf("empty key in chord %q", s)
|
||
}
|
||
if chordMods[part] && main == "" {
|
||
mods = append(mods, part)
|
||
continue
|
||
}
|
||
if main != "" {
|
||
return nil, "", fmt.Errorf("multiple main keys in chord %q", s)
|
||
}
|
||
if namedKeys[part] || len([]rune(part)) == 1 {
|
||
main = part
|
||
continue
|
||
}
|
||
return nil, "", fmt.Errorf("unknown key %q", part)
|
||
}
|
||
if main == "" {
|
||
return nil, "", fmt.Errorf("chord %q has no main key", s)
|
||
}
|
||
return mods, main, nil
|
||
}
|
||
|
||
// chordToXdotool 翻译为 xdotool 的 "mod+keysym" 语法（如 "ctrl+c"、"super+q"）。
|
||
func chordToXdotool(mods []string, main, orig string) (string, error) {
|
||
modMap := map[string]string{
|
||
"ctrl": "ctrl", "control": "ctrl", "alt": "alt", "option": "alt",
|
||
"shift": "shift", "cmd": "super", "command": "super",
|
||
"win": "super", "super": "super", "meta": "super",
|
||
}
|
||
mainMap := map[string]string{
|
||
"enter": "Return", "return": "Return", "escape": "Escape", "esc": "Escape",
|
||
"tab": "Tab", "delete": "BackSpace", "backspace": "BackSpace", "forwarddelete": "Delete",
|
||
"up": "Up", "down": "Down", "left": "Left", "right": "Right",
|
||
"space": "space", "home": "Home", "end": "End",
|
||
"pageup": "Page_Up", "pagedown": "Page_Down",
|
||
}
|
||
parts := make([]string, 0, len(mods)+1)
|
||
for _, m := range mods {
|
||
if m == "fn" {
|
||
return "", fmt.Errorf("fn is not supported on Linux (chord %q)", orig)
|
||
}
|
||
parts = append(parts, modMap[m])
|
||
}
|
||
if sym, ok := mainMap[main]; ok {
|
||
parts = append(parts, sym)
|
||
} else if n := fnKeyNum(main); n > 0 {
|
||
parts = append(parts, "F"+itoa(n))
|
||
} else {
|
||
parts = append(parts, main) // 单字符即 keysym
|
||
}
|
||
return strings.Join(parts, "+"), nil
|
||
}
|
||
|
||
// chordToSendKeys 翻译为 Windows SendKeys 语法（^=ctrl %=alt +=shift）。
|
||
func chordToSendKeys(mods []string, main, orig string) (string, error) {
|
||
modMap := map[string]string{"ctrl": "^", "control": "^", "alt": "%", "option": "%", "shift": "+"}
|
||
prefix := ""
|
||
for _, m := range mods {
|
||
if sym, ok := modMap[m]; ok {
|
||
prefix += sym
|
||
continue
|
||
}
|
||
return "", fmt.Errorf("modifier %q is not supported by Windows SendKeys (chord %q)", m, orig)
|
||
}
|
||
mainMap := map[string]string{
|
||
"enter": "{ENTER}", "return": "{ENTER}", "tab": "{TAB}", "escape": "{ESC}", "esc": "{ESC}",
|
||
"delete": "{BS}", "backspace": "{BS}", "forwarddelete": "{DEL}",
|
||
"up": "{UP}", "down": "{DOWN}", "left": "{LEFT}", "right": "{RIGHT}",
|
||
"space": " ", "home": "{HOME}", "end": "{END}",
|
||
"pageup": "{PGUP}", "pagedown": "{PGDN}",
|
||
}
|
||
if sym, ok := mainMap[main]; ok {
|
||
return prefix + sym, nil
|
||
}
|
||
if n := fnKeyNum(main); n > 0 {
|
||
return prefix + "{F" + itoa(n) + "}", nil
|
||
}
|
||
c := []rune(main)[0]
|
||
if strings.ContainsRune("+^%~(){}[]", c) {
|
||
return prefix + "{" + main + "}", nil
|
||
}
|
||
return prefix + main, nil
|
||
}
|
||
|
||
// sendKeysEscape 把任意文本转义为 SendKeys 字面量（特殊字符包 {}，
|
||
// 换行/制表转 Enter/Tab）。
|
||
func sendKeysEscape(text string) string {
|
||
var b strings.Builder
|
||
for _, r := range text {
|
||
switch r {
|
||
case '\n':
|
||
b.WriteString("{ENTER}")
|
||
case '\t':
|
||
b.WriteString("{TAB}")
|
||
case '+', '^', '%', '~', '(', ')', '{', '}', '[', ']':
|
||
b.WriteString("{" + string(r) + "}")
|
||
default:
|
||
b.WriteRune(r)
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// parseRegion 解析 "x,y,w,h"。
|
||
func parseRegion(region string) (x, y, w, h int, err error) {
|
||
if _, err := fmt.Sscanf(strings.ReplaceAll(region, " ", ""), "%d,%d,%d,%d", &x, &y, &w, &h); err != nil {
|
||
return 0, 0, 0, 0, fmt.Errorf("invalid region %q (want x,y,w,h)", region)
|
||
}
|
||
return x, y, w, h, nil
|
||
}
|
||
|
||
func intArg(m map[string]any, key string) int {
|
||
if v, ok := m[key].(float64); ok {
|
||
return int(v)
|
||
}
|
||
return -1
|
||
}
|
||
|
||
func itoa(n int) string { return fmt.Sprint(n) }
|
||
|
||
// oneLine 把多行命令输出压成单行（用于错误消息）。
|
||
func oneLine(b []byte) string {
|
||
s := string(b)
|
||
for i, r := range s {
|
||
if r == '\n' || r == '\r' {
|
||
s = s[:i]
|
||
break
|
||
}
|
||
}
|
||
return s
|
||
}
|