package tools import ( "context" "net/http" "net/http/httptest" "strings" "testing" ) func TestHTMLToText(t *testing.T) { in := ` Doc Title

Hello

a & b
c

tail text
inline` out := htmlToText([]byte(in)) if !strings.Contains(out, "title: Doc Title") { t.Fatalf("title not extracted: %q", out) } if strings.Contains(out, "alert") || strings.Contains(out, "color:red") { t.Fatalf("script/style leaked: %q", out) } if !strings.Contains(out, "a & b") { t.Fatalf("entity not decoded: %q", out) } // 块级元素分行:Hello 与 a & b 应在不同行。 if !strings.Contains(out, "Hello\na & b") { t.Fatalf("block separation wrong: %q", out) } //   由解析器解码为普通空格,行内文本不粘连;div 为块级元素,前后换行。 if !strings.Contains(out, "tail text\ninline") { t.Fatalf("inline text mangled: %q", out) } } func TestHTMLToTextEmpty(t *testing.T) { if out := htmlToText([]byte("")); out != "" { t.Fatalf("expected empty, got %q", out) } } func TestFetchTool(t *testing.T) { SetAllowedRoots() // 不限制(fetch 不走路径围栏,显式声明意图) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/page": w.Header().Set("Content-Type", "text/html; charset=utf-8") _, _ = w.Write([]byte("ok

hello fetch

")) case "/json": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"a":1}`)) case "/gone": w.WriteHeader(http.StatusNotFound) case "/binary": w.Header().Set("Content-Type", "image/png") _, _ = w.Write([]byte{0x89, 'P', 'N', 'G'}) case "/ua": if ua := r.Header.Get("User-Agent"); !strings.Contains(ua, "Chrome") { w.WriteHeader(http.StatusTeapot) _, _ = w.Write([]byte("ua=" + ua)) return } w.Header().Set("Content-Type", "text/plain") _, _ = w.Write([]byte("chrome impersonated")) default: w.WriteHeader(http.StatusTeapot) } })) defer srv.Close() call := func(url string) (string, error) { f := Fetch() res, err := f.Run(context.Background(), `{"url":"`+url+`"}`) if err != nil { return "", err } return res.Text, nil } // HTML 转文本。 out, err := call(srv.URL + "/page") if err != nil { t.Fatal(err) } if !strings.Contains(out, "title: ok") || !strings.Contains(out, "hello fetch") { t.Fatalf("unexpected page result: %q", out) } // JSON 原样返回。 out, err = call(srv.URL + "/json") if err != nil { t.Fatal(err) } if strings.TrimSpace(out) != `{"a":1}` { t.Fatalf("unexpected json result: %q", out) } // 非 2xx 报错。 if _, err = call(srv.URL + "/gone"); err == nil || !strings.Contains(err.Error(), "404") { t.Fatalf("expected 404 error, got %v", err) } // 二进制内容报错并引导换工具。 if _, err = call(srv.URL + "/binary"); err == nil || !strings.Contains(err.Error(), "browser") { t.Fatalf("expected binary guidance error, got %v", err) } // 指纹仿真生效:默认 Chrome UA。 out, err = call(srv.URL + "/ua") if err != nil { t.Fatal(err) } if out != "chrome impersonated" { t.Fatalf("unexpected ua result: %q", out) } // 非法 URL。 if _, err = call("ftp://x"); err == nil { t.Fatal("expected invalid url error") } }