Sakurasan
661d157b6c
- internal/linkmeta：取正文第一个 http(s) 链接，解析 og: → twitter: → <title>/ description 逐级兜底，解 HTML 实体、相对图片补成绝对地址。按对外抓取设防： 只放行 http(s)、读满 512KB 即停、最多 3 次跳转、整体限时；SSRF 防护做在拨号 那一刻——解析出 IP 后筛掉回环/私网/链路本地/组播/CGNAT 再直连该 IP，堵住 DNS 重绑定窗口 - 字符集转换：按「HTTP 头 charset → <meta charset> → UTF-8」解码，gb2312 归一 成 gbk（超集，按声明解会漏字）；认不出的字符名退回原始字节不报错。为此引入 golang.org/x/text（官方包，约 +1MB 二进制）。tidy 顺带把 aws-sdk 三个包从 indirect 修正为直接依赖——storage/r2.go 本来就直接用它们 - posts 加 link_card 列（JSON，沿用 images 的编解码）；admin 保存短文时重算： 同链接沿用旧卡片不重复打远端，删链接或抓取失败则清空，长文不参与 - 前端 LinkCard 组件接两套 UI 的时间线与详情页，配色写成 vivid 变量优先、 classic 变量兜底，一份样式两边通用；外链图挂了自动隐藏不留空框 - 卡片用 div[role=link] 而非 <a>（vivid 整条短文包在 RouterLink 里，嵌套 <a> 非法），点击必须 preventDefault——只 stop 挡不住祖先 <a> 的默认激活行为
192 lines
5.9 KiB
Go
192 lines
5.9 KiB
Go
package linkmeta
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"net"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"net/url"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
func TestFirstURL(t *testing.T) {
|
||
cases := map[string]string{
|
||
"看看 https://example.com/a?x=1 这个": "https://example.com/a?x=1",
|
||
"[Go 语言](https://go.dev/doc) 官方文档": "https://go.dev/doc",
|
||
"结尾标点要剪掉 https://a.cn/page。": "https://a.cn/page",
|
||
"没有链接就是空": "",
|
||
"两个 https://first.cn https://second.cn": "https://first.cn",
|
||
"裸 www.example.com 不算": "",
|
||
}
|
||
for in, want := range cases {
|
||
if got := FirstURL(in); got != want {
|
||
t.Errorf("FirstURL(%q) = %q, want %q", in, got, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestParsePriority(t *testing.T) {
|
||
doc := `<html><head>
|
||
<title>兜底标题</title>
|
||
<meta name="description" content="兜底描述">
|
||
<meta property="og:title" content="OG 标题">
|
||
<meta content="OG 描述" property="og:description">
|
||
<meta name="og:site_name" content="示例站">
|
||
<meta property="og:image" content="/pics/a.png">
|
||
</head><body><p>正文里的 <b>标签</b> 不该被当成标题</p></body></html>`
|
||
base, _ := url.Parse("https://example.com/post/1")
|
||
c := parse(doc, base)
|
||
|
||
if c.Title != "OG 标题" {
|
||
t.Errorf("title=%q 应优先 og:title", c.Title)
|
||
}
|
||
if c.Desc != "OG 描述" {
|
||
t.Errorf("desc=%q", c.Desc)
|
||
}
|
||
if c.Site != "示例站" {
|
||
t.Errorf("site=%q", c.Site)
|
||
}
|
||
if c.Image != "https://example.com/pics/a.png" {
|
||
t.Errorf("相对图片未补全: %q", c.Image)
|
||
}
|
||
}
|
||
|
||
func TestParseFallsBackToTitleTag(t *testing.T) {
|
||
base, _ := url.Parse("https://example.com/")
|
||
c := parse(`<html><head><title> 只有
|
||
标题 </title></head></html>`, base)
|
||
if c.Title != "只有 标题" {
|
||
t.Errorf("title=%q", c.Title)
|
||
}
|
||
if c.Site != "example.com" {
|
||
t.Errorf("site=%q 应退回主机名", c.Site)
|
||
}
|
||
}
|
||
|
||
func TestParseUnescapesEntities(t *testing.T) {
|
||
base, _ := url.Parse("https://example.com/")
|
||
c := parse(`<head><meta property="og:title" content="Tom & Jerry "quoted""></head>`, base)
|
||
if c.Title != `Tom & Jerry "quoted"` {
|
||
t.Errorf("title=%q", c.Title)
|
||
}
|
||
}
|
||
|
||
func TestBlockedAddresses(t *testing.T) {
|
||
want := map[string]bool{
|
||
"127.0.0.1": true,
|
||
"10.0.3.5": true,
|
||
"172.16.0.1": true,
|
||
"192.168.1.1": true,
|
||
"169.254.169.254": true, // 云元数据
|
||
"100.64.0.1": true, // CGNAT
|
||
"::1": true,
|
||
"fe80::1": true,
|
||
"fc00::1": true, // ULA
|
||
"93.184.216.34": false,
|
||
"2606:2800:220:1:248:1893:25c8:1946": false,
|
||
}
|
||
for s, wantBlocked := range want {
|
||
ip := net.ParseIP(s)
|
||
if ip == nil {
|
||
t.Fatalf("bad test ip %q", s)
|
||
}
|
||
if got := blocked(ip); got != wantBlocked {
|
||
t.Errorf("blocked(%s) = %v, want %v", s, got, wantBlocked)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 默认拨号器必须拒掉回环地址——httptest 的服务就在 127.0.0.1，
|
||
// 这条同时验证了「防护生效」和「测试用的注入通道确实是必要的」。
|
||
func TestDefaultFetchRejectsLoopback(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
fmt.Fprint(w, `<html><head><title>x</title></head></html>`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
_, err := Fetch(context.Background(), srv.URL)
|
||
if err == nil {
|
||
t.Fatal("默认 Fetcher 竟然后到了回环地址，SSRF 防护形同虚设")
|
||
}
|
||
if !strings.Contains(err.Error(), "not a public address") && !strings.Contains(err.Error(), "no public address") {
|
||
t.Fatalf("err=%v，应因内网地址被拒", err)
|
||
}
|
||
}
|
||
|
||
func TestFetcherParsesRealPage(t *testing.T) {
|
||
var gotUA, gotAccept string
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
gotUA, gotAccept = r.Header.Get("User-Agent"), r.Header.Get("Accept")
|
||
if r.URL.Path == "/nope" {
|
||
w.WriteHeader(http.StatusNotFound)
|
||
return
|
||
}
|
||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||
fmt.Fprint(w, `<html><head><meta property="og:title" content="标题"><meta property="og:image" content="https://cdn.example/i.png"></head>`)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
f := &Fetcher{Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||
var d net.Dialer
|
||
return d.DialContext(ctx, network, addr)
|
||
}, Timeout: 2 * time.Second}
|
||
|
||
c, err := f.Fetch(context.Background(), srv.URL+"/page")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if c.Title != "标题" || c.Image != "https://cdn.example/i.png" {
|
||
t.Fatalf("card=%+v", c)
|
||
}
|
||
if !strings.HasPrefix(c.URL, srv.URL) {
|
||
t.Errorf("url=%q", c.URL)
|
||
}
|
||
if gotUA != userAgent {
|
||
t.Errorf("ua=%q", gotUA)
|
||
}
|
||
if !strings.Contains(gotAccept, "text/html") {
|
||
t.Errorf("accept=%q", gotAccept)
|
||
}
|
||
if _, err := f.Fetch(context.Background(), srv.URL+"/nope"); err == nil {
|
||
t.Error("404 应报错")
|
||
}
|
||
}
|
||
|
||
func TestFetchRejectsNonHTMLAndBadScheme(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "application/octet-stream")
|
||
w.Write([]byte("binary"))
|
||
}))
|
||
defer srv.Close()
|
||
|
||
f := &Fetcher{Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||
var d net.Dialer
|
||
return d.DialContext(ctx, network, addr)
|
||
}}
|
||
if _, err := f.Fetch(context.Background(), srv.URL); err == nil {
|
||
t.Error("非 HTML 应拒绝")
|
||
}
|
||
if _, err := f.Fetch(context.Background(), "ftp://example.com/x"); err == nil {
|
||
t.Error("非 http(s) 协议应拒绝")
|
||
}
|
||
if _, err := f.Fetch(context.Background(), "not a url"); err == nil {
|
||
t.Error("坏 URL 应报错")
|
||
}
|
||
}
|
||
|
||
func TestCardEmpty(t *testing.T) {
|
||
if !(&Card{URL: "https://x.cn"}).Empty() {
|
||
t.Error("只有 URL 不算有内容")
|
||
}
|
||
if (&Card{URL: "https://x.cn", Title: "t"}).Empty() {
|
||
t.Error("有标题不该判空")
|
||
}
|
||
var nilCard *Card
|
||
if !nilCard.Empty() {
|
||
t.Error("nil 应判空")
|
||
}
|
||
}
|