Sakurasan
d5dac5a6fc
llm（三协议 Provider）/ session（JSONL 持久化）/ tools（Registry、围栏、 builtin、computer）/ cdp（browser）/ skills / mcp / agent（循环、两级压缩、 编排、审批门控）+ 底座 jsonl/textx/execx/diag/scratch/spill/imagecache。 产品装配代码（config 桥接、schedule 工具、memory/plan/retro/prompt） 留在 AutoGo，宿主接入方式见 MIGRATION.md。
248 lines
8.2 KiB
Go
248 lines
8.2 KiB
Go
package tools
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"net/url"
|
||
"regexp"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/imroc/req/v3"
|
||
"golang.org/x/net/html"
|
||
|
||
"git.gopher.ink/mirrors2/AutoZero/execx"
|
||
)
|
||
|
||
// fetch 工具：纯 HTTP 抓取（req v3 指纹仿真，TLS + HTTP/2 层伪装成真实浏览器），
|
||
// 不依赖本机浏览器——与 browser 工具（CDP）互补：fetch 毫秒级返回可读文本，
|
||
// 需要 JS 渲染、登录态或截图时才换 browser。指纹 profile 与代理由配置注入
|
||
// （SetFetchDefaults），未配置时默认 Chrome 仿真、直连。
|
||
|
||
const (
|
||
fetchTimeout = 20 * time.Second
|
||
fetchMaxBytes = 2 << 20 // 响应体读取上限（防超大文件）
|
||
fetchMaxChars = 16000 // 返回给模型的文本上限（与 bash 截断同量级）
|
||
)
|
||
|
||
var (
|
||
fetchMu sync.Mutex
|
||
fetchProfile string
|
||
fetchProxy string
|
||
fetchOnce sync.Once
|
||
fetchClient *req.Client
|
||
)
|
||
|
||
// SetFetchDefaults 在配置装载后、首次 fetch 前调用：设置指纹仿真 profile
|
||
// （chrome | firefox | safari，未知值回退 chrome）与代理 URL。
|
||
func SetFetchDefaults(profile, proxy string) {
|
||
fetchMu.Lock()
|
||
defer fetchMu.Unlock()
|
||
fetchProfile, fetchProxy = profile, proxy
|
||
}
|
||
|
||
func fetchHTTPClient() *req.Client {
|
||
fetchMu.Lock()
|
||
profile, proxy := fetchProfile, fetchProxy
|
||
fetchMu.Unlock()
|
||
fetchOnce.Do(func() {
|
||
c := req.C().SetTimeout(fetchTimeout)
|
||
switch strings.ToLower(strings.TrimSpace(profile)) {
|
||
case "firefox":
|
||
c.ImpersonateFirefox()
|
||
case "safari":
|
||
c.ImpersonateSafari()
|
||
default:
|
||
c.ImpersonateChrome()
|
||
}
|
||
if proxy != "" {
|
||
c.SetProxyURL(proxy)
|
||
}
|
||
// 响应体上限：超限截断而非报错，让模型至少看到开头内容。
|
||
// 按字符边界退：下游 renderBody 之后还有一道 execx.Truncate(fetchMaxChars)，
|
||
// 所以这一处今天不会把半个字露到模型眼前——但"先夹字节、后面自然有人兜"
|
||
// 正是这类 bug 长回来的地方，而这里改一次只需要换个函数。
|
||
c.SetResponseBodyTransformer(func(raw []byte, _ *req.Request, _ *req.Response) ([]byte, error) {
|
||
if len(raw) > fetchMaxBytes {
|
||
raw, _ = execx.Truncate(raw, fetchMaxBytes)
|
||
}
|
||
return raw, nil
|
||
})
|
||
fetchClient = c
|
||
})
|
||
return fetchClient
|
||
}
|
||
|
||
// Fetch 返回 fetch 工具（GET-only：只读语义的网络访问，审批按 network 类走）。
|
||
func Fetch() *Tool {
|
||
return &Tool{
|
||
Name: "fetch",
|
||
Description: "Fetch a URL over HTTP (TLS/HTTP2 fingerprint impersonates a real browser; no local browser needed) and return readable text: HTML is converted to plain text, JSON/XML/markdown returned as-is. GET only, follows redirects. " +
|
||
"Use it as the fast default for reading web pages, docs and APIs — it works even where headless Chrome is not installed. " +
|
||
"Switch to the `browser` tool only when the page needs JavaScript rendering, login state, or a screenshot; use bash curl to download binary files.",
|
||
SideEffect: true,
|
||
Kind: KindNetwork, // 访问外部网络
|
||
Parameters: map[string]any{
|
||
"type": "object",
|
||
"properties": map[string]any{
|
||
"url": map[string]any{"type": "string", "description": "Absolute http/https URL to fetch"},
|
||
},
|
||
"required": []string{"url"},
|
||
},
|
||
Run: func(ctx context.Context, input string) (Result, error) {
|
||
m, err := arg(input)
|
||
if err != nil {
|
||
return Result{}, err
|
||
}
|
||
u := strings.TrimSpace(str(m, "url"))
|
||
pu, perr := url.Parse(u)
|
||
if u == "" || perr != nil || (pu.Scheme != "http" && pu.Scheme != "https") {
|
||
return Result{}, fmt.Errorf("invalid url %q (absolute http/https required)", u)
|
||
}
|
||
rctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||
defer cancel()
|
||
resp, err := fetchHTTPClient().R().SetContext(rctx).Get(u)
|
||
if err != nil {
|
||
return Result{}, err
|
||
}
|
||
if resp.StatusCode >= 400 {
|
||
return Result{}, fmt.Errorf("HTTP %d for %s", resp.StatusCode, u)
|
||
}
|
||
text := renderBody(resp.GetContentType(), resp.Bytes())
|
||
if text == "" {
|
||
return Result{}, fmt.Errorf("no readable content (content-type %q) — for rendered pages use the browser tool, for binaries use bash curl", resp.GetContentType())
|
||
}
|
||
// 按字符边界截（原实现 out[:16000] 会把汉字劈成半个字节），并把
|
||
// "被截过"结构化标出来——模型看到的不是全量，UI 与统计都该知道。
|
||
raw, truncated := execx.Truncate([]byte(text), fetchMaxChars)
|
||
if truncated {
|
||
raw = append(raw, "\n...[truncated]"...)
|
||
}
|
||
return Result{Text: string(raw), Truncated: truncated}, nil
|
||
},
|
||
}
|
||
}
|
||
|
||
// renderBody 按内容类型把响应体转为可读文本：HTML 转纯文本，文本类原样，
|
||
// 二进制返回空串（由调用方报错引导换工具）。Content-Type 缺失时按内容嗅探。
|
||
func renderBody(contentType string, body []byte) string {
|
||
if strings.HasPrefix(contentType, "image/") || strings.HasPrefix(contentType, "audio/") ||
|
||
strings.HasPrefix(contentType, "video/") || strings.Contains(contentType, "octet-stream") ||
|
||
strings.Contains(contentType, "font") {
|
||
return ""
|
||
}
|
||
if isHTMLType(contentType, body) {
|
||
return htmlToText(body)
|
||
}
|
||
if isTextType(contentType) {
|
||
return string(body)
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func isHTMLType(ct string, body []byte) bool {
|
||
if strings.Contains(ct, "html") {
|
||
return true
|
||
}
|
||
// 无 Content-Type（或通用 application/octet-stream）时按内容嗅探。
|
||
head := body
|
||
if len(head) > 512 {
|
||
// 只是嗅探，切坏也不会露到用户面前；但这里紧跟着就 `string(head)`，
|
||
// 留一个非法字节序列在字符串里，下一个改这个函数的人会以为是安全的。
|
||
head, _ = execx.Truncate(head, 512)
|
||
}
|
||
low := strings.ToLower(string(head))
|
||
return strings.HasPrefix(low, "<!doctype html") || strings.Contains(low, "<html")
|
||
}
|
||
|
||
func isTextType(ct string) bool {
|
||
if strings.HasPrefix(ct, "text/") {
|
||
return true
|
||
}
|
||
for _, t := range []string{"json", "xml", "javascript", "yaml", "csv"} {
|
||
if strings.Contains(ct, t) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// ---- HTML → 可读文本 ----
|
||
|
||
// fetchSkipElems 整棵剪掉：对可读文本是纯噪声的元素。
|
||
var fetchSkipElems = map[string]bool{
|
||
"script": true, "style": true, "noscript": true, "template": true, "svg": true,
|
||
}
|
||
|
||
// fetchBlockElems 块级元素：前后补换行，让文本按视觉块分行。
|
||
var fetchBlockElems = map[string]bool{
|
||
"p": true, "div": true, "br": true, "li": true, "tr": true, "table": true,
|
||
"ul": true, "ol": true, "blockquote": true, "pre": true, "form": true,
|
||
"h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true,
|
||
"header": true, "footer": true, "main": true, "section": true, "article": true,
|
||
"figure": true, "figcaption": true, "hr": true, "option": true, "dd": true, "dt": true,
|
||
}
|
||
|
||
// \p{Zs} 覆盖 等解码出的 Unicode 空格（不在 Go 的 \s 字符类里）。
|
||
var fetchSpaceRe = regexp.MustCompile(`[\s\p{Zs}]+`)
|
||
|
||
// htmlToText 把 HTML 转成可读文本：<title> 提前，块级元素分行，
|
||
// 行内空白压缩为单空格，实体由解析器解码。
|
||
func htmlToText(body []byte) string {
|
||
root, err := html.Parse(strings.NewReader(string(body)))
|
||
if err != nil {
|
||
return string(body)
|
||
}
|
||
var title, b strings.Builder
|
||
var walk func(n *html.Node)
|
||
walk = func(n *html.Node) {
|
||
if n.Type == html.ElementNode {
|
||
name := strings.ToLower(n.Data)
|
||
switch {
|
||
case name == "title":
|
||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||
if c.Type == html.TextNode {
|
||
title.WriteString(fetchSpaceRe.ReplaceAllString(c.Data, " "))
|
||
}
|
||
}
|
||
return
|
||
case fetchSkipElems[name]:
|
||
return
|
||
case fetchBlockElems[name]:
|
||
b.WriteString("\n")
|
||
}
|
||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||
walk(c)
|
||
}
|
||
if fetchBlockElems[name] {
|
||
b.WriteString("\n")
|
||
}
|
||
return
|
||
}
|
||
if n.Type == html.TextNode {
|
||
b.WriteString(fetchSpaceRe.ReplaceAllString(n.Data, " "))
|
||
}
|
||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||
walk(c)
|
||
}
|
||
}
|
||
walk(root)
|
||
|
||
// 逐行去首尾空白、去空行：源码缩进与连续空行是噪声。
|
||
var lines []string
|
||
for _, line := range strings.Split(b.String(), "\n") {
|
||
if line = strings.TrimSpace(line); line != "" {
|
||
lines = append(lines, line)
|
||
}
|
||
}
|
||
if len(lines) == 0 {
|
||
return ""
|
||
}
|
||
text := strings.Join(lines, "\n")
|
||
if t := strings.TrimSpace(title.String()); t != "" {
|
||
return "title: " + t + "\n\n" + text
|
||
}
|
||
return text
|
||
}
|