feat: AutoZero 通用 Agent 框架（从 AutoGo 五阶段拆分完成）
llm（三协议 Provider）/ session（JSONL 持久化）/ tools（Registry、围栏、 builtin、computer）/ cdp（browser）/ skills / mcp / agent（循环、两级压缩、 编排、审批门控）+ 底座 jsonl/textx/execx/diag/scratch/spill/imagecache。 产品装配代码（config 桥接、schedule 工具、memory/plan/retro/prompt） 留在 AutoGo，宿主接入方式见 MIGRATION.md。
This commit is contained in:
Sakurasan committed
commit
d5dac5a6fc
140 files changed
+26497
No files matched your search
@@ -0,0 +1,331 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.gopher.ink/mirrors2/AutoZero/llm"
|
||||
)
|
||||
|
||||
// computer 工具：本地桌面控制（借鉴 Claude Code / ccb 的 Computer Use）。
|
||||
// 平台实现由 guiBackend 抽象、编译期选择，各自独立成文件：
|
||||
// - computer_darwin.go macOS：osascript(System Events) + screencapture，
|
||||
// 检测到 cliclick 时点击更可靠（保持独立的实现）；
|
||||
// - computer_linux.go Linux：xdotool + scrot/import/gnome-screenshot（X11，Wayland 不支持）；
|
||||
// - computer_windows.go Windows：PowerShell（SendKeys 键盘 + user32 P/Invoke 鼠标/滚轮）。
|
||||
//
|
||||
// 分发与参数校验在本文件（共享）；校验失败必须发生在任何真实 GUI 调用之前。
|
||||
|
||||
// guiBackend 是单平台的桌面控制实现。
|
||||
type guiBackend interface {
|
||||
Available() error // 依赖检查（缺工具时给出可操作的报错）
|
||||
Screenshot(ctx context.Context, region string) (string, error) // region 为 "x,y,w,h" 或 ""
|
||||
Click(ctx context.Context, x, y int, kind string) error // kind: left / right / double
|
||||
TypeText(ctx context.Context, text string) error
|
||||
KeyChord(ctx context.Context, chordStr string) error
|
||||
Scroll(ctx context.Context, direction string, amount int) error
|
||||
PreflightPermissions() []string // 首次运行引导授予桌面权限；返回缺失项（空=齐全）
|
||||
}
|
||||
|
||||
// PreflightDesktopPermissions 探测并（首次运行）引导授予桌面控制所需的系统权限
|
||||
// （macOS 的辅助功能 / 屏幕录制）。返回缺失权限名（空切片=齐全）。非 macOS 由 backend
|
||||
// 直接返回 nil。在程序启动时调用一次，缺失时自动打开系统设置面板并打印指引。
|
||||
func PreflightDesktopPermissions() []string {
|
||||
return currentGuiBackend.PreflightPermissions()
|
||||
}
|
||||
|
||||
// currentGuiBackend 由各平台的 newGuiBackend() 在编译期决定。
|
||||
var currentGuiBackend = newGuiBackend()
|
||||
|
||||
// Computer 返回 computer 工具（三平台通用入口）。
|
||||
func Computer() *Tool {
|
||||
return &Tool{
|
||||
Name: "computer",
|
||||
Description: "Control the local desktop: screenshot (returned as an image you can see), click, double-click, right-click, type text, press key chords, scroll. " +
|
||||
"macOS: needs Accessibility & Screen Recording permission. If a click fails with an accessibility/osascript error (-25211, -1728), STOP retrying clicks and switch to `osascript` (AppleScript/System Events) or the `browser` tool instead. " +
|
||||
"Screenshots are expensive (vision inference ~10-70s each): take one only when you must SEE pixels (e.g. final confirmation for the user); for verification prefer programmatic state reads (browser `js`, `osascript` reading menu/window state). " +
|
||||
"Click coordinates are in real logical screen resolution — do not trust coordinates derived from a screenshot's displayed size. " +
|
||||
"Linux: needs xdotool + scrot/import under X11 (Wayland unsupported). " +
|
||||
"Windows: uses PowerShell (Win key unsupported by SendKeys).",
|
||||
SideEffect: true,
|
||||
Kind: KindExec, // 控制本机桌面：执行类
|
||||
Parameters: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"action": map[string]any{"type": "string", "enum": []string{"screenshot", "left_click", "right_click", "double_click", "type", "key", "scroll"},
|
||||
"description": "GUI action to perform"},
|
||||
"x": map[string]any{"type": "integer", "description": "Screen x coordinate (click actions)"},
|
||||
"y": map[string]any{"type": "integer", "description": "Screen y coordinate (click actions)"},
|
||||
"text": map[string]any{"type": "string", "description": "Text to type (action=type)"},
|
||||
"key": map[string]any{"type": "string", "description": "Key or chord, e.g. enter, cmd+c / ctrl+c, cmd+shift+t (action=key)"},
|
||||
"direction": map[string]any{"type": "string", "enum": []string{"up", "down"},
|
||||
"description": "Scroll direction (action=scroll, default down)"},
|
||||
"amount": map[string]any{"type": "integer", "description": "Scroll repetitions 1-10 (action=scroll, default 3)"},
|
||||
},
|
||||
"required": []string{"action"},
|
||||
},
|
||||
Run: func(ctx context.Context, input string) (Result, error) {
|
||||
m, err := arg(input)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
b := currentGuiBackend
|
||||
if err := b.Available(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
vision := func(path string, err error) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
p, serr := ShrinkForVision(path)
|
||||
if serr != nil {
|
||||
return path, nil
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
action := str(m, "action")
|
||||
switch action {
|
||||
case "screenshot":
|
||||
path, err := vision(b.Screenshot(ctx, ""))
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{Text: "screenshot saved", Images: []llm.Image{{Path: path}}}, nil
|
||||
|
||||
case "left_click", "right_click", "double_click":
|
||||
x, y := intArg(m, "x"), intArg(m, "y")
|
||||
if x < 0 || y < 0 {
|
||||
return Result{}, fmt.Errorf("action %q requires valid x,y coordinates", action)
|
||||
}
|
||||
kind := strings.TrimSuffix(action, "_click")
|
||||
if err := b.Click(ctx, x, y, kind); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
// 操作后截图验证；截图失败降级为纯文本，不让点击白做。
|
||||
path, serr := vision(b.Screenshot(ctx, ""))
|
||||
if serr != nil {
|
||||
return TextResult(action + " at " + itoa(x) + "," + itoa(y) + " (post-action screenshot failed: " + serr.Error() + ")"), nil
|
||||
}
|
||||
return Result{Text: action + " at " + itoa(x) + "," + itoa(y), Images: []llm.Image{{Path: path}}}, nil
|
||||
|
||||
case "type":
|
||||
text := str(m, "text")
|
||||
if text == "" {
|
||||
return Result{}, fmt.Errorf("action type requires non-empty text")
|
||||
}
|
||||
if err := b.TypeText(ctx, text); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return TextResult("typed " + itoa(len([]rune(text))) + " chars"), nil
|
||||
|
||||
case "key":
|
||||
chordStr := str(m, "key")
|
||||
if _, _, err := parseChord(chordStr); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := b.KeyChord(ctx, chordStr); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return TextResult("pressed " + chordStr), nil
|
||||
|
||||
case "scroll":
|
||||
dir := str(m, "direction")
|
||||
if dir == "" {
|
||||
dir = "down"
|
||||
}
|
||||
n := intArg(m, "amount")
|
||||
if n <= 0 {
|
||||
n = 3
|
||||
}
|
||||
if n > 10 {
|
||||
n = 10
|
||||
}
|
||||
if err := b.Scroll(ctx, dir, n); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
path, serr := vision(b.Screenshot(ctx, ""))
|
||||
if serr != nil {
|
||||
return TextResult("scrolled " + dir + " (post-action screenshot failed: " + serr.Error() + ")"), nil
|
||||
}
|
||||
return Result{Text: "scrolled " + dir, Images: []llm.Image{{Path: path}}}, nil
|
||||
|
||||
default:
|
||||
return Result{}, fmt.Errorf("unknown action %q", action)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 共享键序模型：统一词表，各平台后端再翻译为自己的语法 ----
|
||||
|
||||
var chordMods = map[string]bool{
|
||||
"cmd": true, "command": true, "ctrl": true, "control": true, "alt": true,
|
||||
"option": true, "shift": true, "fn": true, "win": true, "super": true, "meta": true,
|
||||
}
|
||||
|
||||
var namedKeys = map[string]bool{
|
||||
"enter": true, "return": true, "tab": true, "escape": true, "esc": true,
|
||||
"delete": true, "backspace": true, "forwarddelete": true,
|
||||
"up": true, "down": true, "left": true, "right": true,
|
||||
"space": true, "home": true, "end": true, "pageup": true, "pagedown": true,
|
||||
"f1": true, "f2": true, "f3": true, "f4": true, "f5": true, "f6": true,
|
||||
"f7": true, "f8": true, "f9": true, "f10": true, "f11": true, "f12": true,
|
||||
}
|
||||
|
||||
// fnKeyNum 返回功能键的编号（f1..f12），非功能键返回 0。
|
||||
func fnKeyNum(main string) int {
|
||||
if len(main) >= 2 && main[0] == 'f' {
|
||||
n := 0
|
||||
for _, r := range main[1:] {
|
||||
if r < '0' || r > '9' {
|
||||
return 0
|
||||
}
|
||||
n = n*10 + int(r-'0')
|
||||
}
|
||||
if n >= 1 && n <= 12 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseChord 把 "cmd+shift+t" 解析为修饰键 + 主键（命名键或单字符）。
|
||||
// 全平台共用词表；平台不支持的键由后端在运行时报错。
|
||||
func parseChord(s string) (mods []string, main string, err error) {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return nil, "", fmt.Errorf("empty key chord")
|
||||
}
|
||||
for _, part := range strings.Split(strings.ToLower(strings.TrimSpace(s)), "+") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
return nil, "", fmt.Errorf("empty key in chord %q", s)
|
||||
}
|
||||
if chordMods[part] && main == "" {
|
||||
mods = append(mods, part)
|
||||
continue
|
||||
}
|
||||
if main != "" {
|
||||
return nil, "", fmt.Errorf("multiple main keys in chord %q", s)
|
||||
}
|
||||
if namedKeys[part] || len([]rune(part)) == 1 {
|
||||
main = part
|
||||
continue
|
||||
}
|
||||
return nil, "", fmt.Errorf("unknown key %q", part)
|
||||
}
|
||||
if main == "" {
|
||||
return nil, "", fmt.Errorf("chord %q has no main key", s)
|
||||
}
|
||||
return mods, main, nil
|
||||
}
|
||||
|
||||
// chordToXdotool 翻译为 xdotool 的 "mod+keysym" 语法（如 "ctrl+c"、"super+q"）。
|
||||
func chordToXdotool(mods []string, main, orig string) (string, error) {
|
||||
modMap := map[string]string{
|
||||
"ctrl": "ctrl", "control": "ctrl", "alt": "alt", "option": "alt",
|
||||
"shift": "shift", "cmd": "super", "command": "super",
|
||||
"win": "super", "super": "super", "meta": "super",
|
||||
}
|
||||
mainMap := map[string]string{
|
||||
"enter": "Return", "return": "Return", "escape": "Escape", "esc": "Escape",
|
||||
"tab": "Tab", "delete": "BackSpace", "backspace": "BackSpace", "forwarddelete": "Delete",
|
||||
"up": "Up", "down": "Down", "left": "Left", "right": "Right",
|
||||
"space": "space", "home": "Home", "end": "End",
|
||||
"pageup": "Page_Up", "pagedown": "Page_Down",
|
||||
}
|
||||
parts := make([]string, 0, len(mods)+1)
|
||||
for _, m := range mods {
|
||||
if m == "fn" {
|
||||
return "", fmt.Errorf("fn is not supported on Linux (chord %q)", orig)
|
||||
}
|
||||
parts = append(parts, modMap[m])
|
||||
}
|
||||
if sym, ok := mainMap[main]; ok {
|
||||
parts = append(parts, sym)
|
||||
} else if n := fnKeyNum(main); n > 0 {
|
||||
parts = append(parts, "F"+itoa(n))
|
||||
} else {
|
||||
parts = append(parts, main) // 单字符即 keysym
|
||||
}
|
||||
return strings.Join(parts, "+"), nil
|
||||
}
|
||||
|
||||
// chordToSendKeys 翻译为 Windows SendKeys 语法（^=ctrl %=alt +=shift）。
|
||||
func chordToSendKeys(mods []string, main, orig string) (string, error) {
|
||||
modMap := map[string]string{"ctrl": "^", "control": "^", "alt": "%", "option": "%", "shift": "+"}
|
||||
prefix := ""
|
||||
for _, m := range mods {
|
||||
if sym, ok := modMap[m]; ok {
|
||||
prefix += sym
|
||||
continue
|
||||
}
|
||||
return "", fmt.Errorf("modifier %q is not supported by Windows SendKeys (chord %q)", m, orig)
|
||||
}
|
||||
mainMap := map[string]string{
|
||||
"enter": "{ENTER}", "return": "{ENTER}", "tab": "{TAB}", "escape": "{ESC}", "esc": "{ESC}",
|
||||
"delete": "{BS}", "backspace": "{BS}", "forwarddelete": "{DEL}",
|
||||
"up": "{UP}", "down": "{DOWN}", "left": "{LEFT}", "right": "{RIGHT}",
|
||||
"space": " ", "home": "{HOME}", "end": "{END}",
|
||||
"pageup": "{PGUP}", "pagedown": "{PGDN}",
|
||||
}
|
||||
if sym, ok := mainMap[main]; ok {
|
||||
return prefix + sym, nil
|
||||
}
|
||||
if n := fnKeyNum(main); n > 0 {
|
||||
return prefix + "{F" + itoa(n) + "}", nil
|
||||
}
|
||||
c := []rune(main)[0]
|
||||
if strings.ContainsRune("+^%~(){}[]", c) {
|
||||
return prefix + "{" + main + "}", nil
|
||||
}
|
||||
return prefix + main, nil
|
||||
}
|
||||
|
||||
// sendKeysEscape 把任意文本转义为 SendKeys 字面量（特殊字符包 {}，
|
||||
// 换行/制表转 Enter/Tab）。
|
||||
func sendKeysEscape(text string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range text {
|
||||
switch r {
|
||||
case '\n':
|
||||
b.WriteString("{ENTER}")
|
||||
case '\t':
|
||||
b.WriteString("{TAB}")
|
||||
case '+', '^', '%', '~', '(', ')', '{', '}', '[', ']':
|
||||
b.WriteString("{" + string(r) + "}")
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// parseRegion 解析 "x,y,w,h"。
|
||||
func parseRegion(region string) (x, y, w, h int, err error) {
|
||||
if _, err := fmt.Sscanf(strings.ReplaceAll(region, " ", ""), "%d,%d,%d,%d", &x, &y, &w, &h); err != nil {
|
||||
return 0, 0, 0, 0, fmt.Errorf("invalid region %q (want x,y,w,h)", region)
|
||||
}
|
||||
return x, y, w, h, nil
|
||||
}
|
||||
|
||||
func intArg(m map[string]any, key string) int {
|
||||
if v, ok := m[key].(float64); ok {
|
||||
return int(v)
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func itoa(n int) string { return fmt.Sprint(n) }
|
||||
|
||||
// oneLine 把多行命令输出压成单行（用于错误消息）。
|
||||
func oneLine(b []byte) string {
|
||||
s := string(b)
|
||||
for i, r := range s {
|
||||
if r == '\n' || r == '\r' {
|
||||
s = s[:i]
|
||||
break
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in new issue
Block a user