短文支持链接预览卡片:新增 linkmeta 抓取 + 字符集解码

- internal/linkmeta:取正文第一个 http(s) 链接,解析 og: → twitter: → <title>/
  description 逐级兜底,解 HTML 实体、相对图片补成绝对地址。按对外抓取设防:
  只放行 http(s)、读满 512KB 即停、最多 3 次跳转、整体限时;SSRF 防护做在拨号
  那一刻——解析出 IP 后筛掉回环/私网/链路本地/组播/CGNAT 再直连该 IP,堵住
  DNS 重绑定窗口
- 字符集转换:按「HTTP 头 charset → <meta charset> → UTF-8」解码,gb2312 归一
  成 gbk(超集,按声明解会漏字);认不出的字符名退回原始字节不报错。为此引入
  golang.org/x/text(官方包,约 +1MB 二进制)。tidy 顺带把 aws-sdk 三个包从
  indirect 修正为直接依赖——storage/r2.go 本来就直接用它们
- posts 加 link_card 列(JSON,沿用 images 的编解码);admin 保存短文时重算:
  同链接沿用旧卡片不重复打远端,删链接或抓取失败则清空,长文不参与
- 前端 LinkCard 组件接两套 UI 的时间线与详情页,配色写成 vivid 变量优先、
  classic 变量兜底,一份样式两边通用;外链图挂了自动隐藏不留空框
- 卡片用 div[role=link] 而非 <a>(vivid 整条短文包在 RouterLink 里,嵌套 <a>
  非法),点击必须 preventDefault——只 stop 挡不住祖先 <a> 的默认激活行为
This commit is contained in:
Sakurasan
2026-09-28 17:13:20 +08:00
parent 232abf2019
commit 661d157b6c
17 changed files with 1081 additions and 16 deletions
+5 -4
View File
@@ -3,16 +3,18 @@ module oneblog
go 1.24
require (
github.com/aws/aws-sdk-go-v2 v1.47.1
github.com/aws/aws-sdk-go-v2/config v1.33.6
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.4
github.com/lib/pq v1.10.9
github.com/yuin/goldmark v1.7.13
golang.org/x/text v0.21.0
modernc.org/sqlite v1.39.0
)
require (
github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
github.com/aws/aws-sdk-go-v2/config v1.33.6 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
@@ -21,7 +23,6 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.4 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
+2
View File
@@ -59,6 +59,8 @@ golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM=
+7
View File
@@ -222,6 +222,7 @@ func (a *API) listPosts(w http.ResponseWriter, r *http.Request) {
return
}
in.Status = store.NormalizeStatus(in.Status)
attachLinkCard(r.Context(), &in, nil)
p, err := a.Store.Create(in)
if err != nil {
httpx.ServerError(w, err)
@@ -261,6 +262,12 @@ func (a *API) postByID(w http.ResponseWriter, r *http.Request) {
if in.Status != "" {
in.Status = store.NormalizeStatus(in.Status)
}
// 多读一次当前行只为拿到已有卡片:链接没变就不必再打远端
var curCard *model.LinkCard
if cur, err := a.Store.Get(id); err == nil {
curCard = cur.LinkCard
}
attachLinkCard(r.Context(), &in, curCard)
p, err := a.Store.Update(id, in)
writeOne(w, p, err)
case http.MethodDelete:
+45
View File
@@ -0,0 +1,45 @@
package admin
import (
"context"
"time"
"oneblog/internal/linkmeta"
"oneblog/internal/model"
)
// linkFetchBudget 是给外链抓取的时间预算:发布不该被一个慢站点拖住,
// 超时就当作「这篇没有卡片」。
const linkFetchBudget = 2500 * time.Millisecond
// linkFetch 是要不要抓远端的注入点:默认走带 SSRF 防护的真实抓取,
// 测试里换成假实现(真实抓取在测试环境会被自己的防护拦掉)。
var linkFetch = linkmeta.Fetch
// attachLinkCard 按正文重算短文链接卡片,结果写进 in.LinkCard。
//
// 只处理短文(长文在时间线已有封面媒体位,再加卡片会抢位置);
// 链接没变就沿用旧卡片,不重复抓远端;正文里删掉链接、或抓取失败,
// 一律传空卡片把旧的清掉——宁可没有卡片,也不要挂着别的地址抓来的旧数据。
func attachLinkCard(ctx context.Context, in *model.PostInput, cur *model.LinkCard) {
if in.Kind != model.KindShort || in.ContentMd == "" {
return
}
u := linkmeta.FirstURL(in.ContentMd)
if u == "" {
in.LinkCard = &model.LinkCard{}
return
}
if cur != nil && cur.URL == u {
in.LinkCard = cur
return
}
cctx, cancel := context.WithTimeout(ctx, linkFetchBudget)
defer cancel()
card, err := linkFetch(cctx, u)
if err != nil {
in.LinkCard = &model.LinkCard{}
return
}
in.LinkCard = card
}
+112
View File
@@ -0,0 +1,112 @@
package admin
import (
"context"
"errors"
"testing"
"oneblog/internal/model"
"oneblog/internal/store"
)
func TestAttachLinkCard(t *testing.T) {
old := linkFetch
t.Cleanup(func() { linkFetch = old })
t.Run("长文不动", func(t *testing.T) {
called := false
linkFetch = func(context.Context, string) (*model.LinkCard, error) {
called = true
return nil, nil
}
in := &model.PostInput{Kind: model.KindLong, ContentMd: "看 https://a.cn"}
attachLinkCard(context.Background(), in, nil)
if in.LinkCard != nil || called {
t.Fatalf("长文不该抓取,card=%+v called=%v", in.LinkCard, called)
}
})
t.Run("没链接则清空", func(t *testing.T) {
linkFetch = func(context.Context, string) (*model.LinkCard, error) {
t.Fatal("没链接不该抓")
return nil, nil
}
in := &model.PostInput{Kind: model.KindShort, ContentMd: "纯文本一句"}
attachLinkCard(context.Background(), in, &model.LinkCard{URL: "https://old.cn", Title: "旧"})
if in.LinkCard == nil || !in.LinkCard.Empty() {
t.Fatalf("应传空卡片清掉旧的,got %+v", in.LinkCard)
}
})
t.Run("链接未变则沿用不重抓", func(t *testing.T) {
linkFetch = func(context.Context, string) (*model.LinkCard, error) {
t.Fatal("同一个链接不该重复抓")
return nil, nil
}
cur := &model.LinkCard{URL: "https://a.cn", Title: "已抓过"}
in := &model.PostInput{Kind: model.KindShort, ContentMd: "看 https://a.cn"}
attachLinkCard(context.Background(), in, cur)
if in.LinkCard != cur {
t.Fatalf("应沿用旧卡片,got %+v", in.LinkCard)
}
})
t.Run("新链接则抓取", func(t *testing.T) {
var gotURL string
linkFetch = func(_ context.Context, u string) (*model.LinkCard, error) {
gotURL = u
return &model.LinkCard{URL: u, Title: "新卡片"}, nil
}
in := &model.PostInput{Kind: model.KindShort, ContentMd: "换成 https://b.cn/x。"}
attachLinkCard(context.Background(), in, &model.LinkCard{URL: "https://a.cn", Title: "旧"})
if gotURL != "https://b.cn/x" {
t.Fatalf("结尾标点没剪掉: %q", gotURL)
}
if in.LinkCard == nil || in.LinkCard.Title != "新卡片" {
t.Fatalf("card=%+v", in.LinkCard)
}
})
t.Run("抓取失败则清空", func(t *testing.T) {
linkFetch = func(context.Context, string) (*model.LinkCard, error) {
return nil, errors.New("timeout")
}
in := &model.PostInput{Kind: model.KindShort, ContentMd: "https://slow.cn"}
attachLinkCard(context.Background(), in, &model.LinkCard{URL: "https://a.cn", Title: "旧"})
if in.LinkCard == nil || !in.LinkCard.Empty() {
t.Fatalf("失败应清空,got %+v", in.LinkCard)
}
})
}
func TestLinkCardRoundTrip(t *testing.T) {
a, _ := newTestAPI(t)
card := &model.LinkCard{URL: "https://example.cn/x", Title: "示例标题", Desc: "描述", Site: "example", Image: "https://cdn/i.png"}
p, err := a.Store.Create(model.PostInput{Kind: model.KindShort, ContentMd: "看 https://example.cn/x",
Status: model.StatusPublished, LinkCard: card})
if err != nil {
t.Fatal(err)
}
if p.LinkCard == nil || p.LinkCard.Title != "示例标题" || p.LinkCard.Image != "https://cdn/i.png" {
t.Fatalf("创建后没读回卡片: %+v", p.LinkCard)
}
// 列表接口也要带上(时间线靠它渲染)
page, err := a.Store.List(store.ListOptions{Page: 1, Size: 10})
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 1 || page.Items[0].LinkCard == nil {
t.Fatalf("列表未带卡片: %+v", page.Items)
}
// 空卡片 = 清空
if _, err := a.Store.Update(p.ID, model.PostInput{LinkCard: &model.LinkCard{}}); err != nil {
t.Fatal(err)
}
got, err := a.Store.Get(p.ID)
if err != nil {
t.Fatal(err)
}
if got.LinkCard != nil {
t.Fatalf("应已清空,got %+v", got.LinkCard)
}
}
+107
View File
@@ -0,0 +1,107 @@
package linkmeta
import (
"bytes"
"strings"
"golang.org/x/text/encoding/htmlindex"
"golang.org/x/text/transform"
)
// charset.go 负责把响应字节解成 UTF-8 字符串。
//
// 为什么必须做:老派中文站点(以及不少论坛/院校站)仍在用 GBK/GB2312/Big5,
// 直接当 UTF-8 读会得到一串替换字符,卡片标题就成了乱码。与其显示乱码,
// 不如抓对——这是纯展示层的事,不该让站主去改对方的编码。
// decode 按「HTTP 头声明 → 文档里的 <meta charset> → UTF-8」的优先级解码。
// 已经是 UTF-8/ASCII 时原样返回,不绕转换管线。
func decode(raw []byte, contentType string) string {
raw = bytes.TrimPrefix(raw, []byte("\ufeff")) // UTF-8 BOM
name := charsetFromHeader(contentType)
if name == "" {
// meta 标签本身是 ASCII(GBK/Big5 都是 ASCII 超集),
// 所以从未解码的原始字节里嗅探是安全的,只看文档开头。
name = charsetFromMeta(raw)
}
if name == "" || isUTF8(name) {
return string(raw)
}
enc, err := htmlindex.Get(name)
if err != nil {
return string(raw) // 没见过的字符名:按 UTF-8 尽力而为
}
out, _, err := transform.Bytes(enc.NewDecoder(), raw)
if err != nil && len(out) == 0 {
return string(raw)
}
return string(out)
}
func charsetFromHeader(ct string) string {
for _, part := range strings.Split(ct, ";") {
kv := strings.SplitN(strings.TrimSpace(part), "=", 2)
if len(kv) == 2 && strings.EqualFold(kv[0], "charset") {
return normalizeCharset(strings.Trim(kv[1], `"'`))
}
}
return ""
}
var (
metaCharset = []byte("charset=")
)
// charsetFromMeta 在文档开头找 <meta charset="x"> 或
// <meta http-equiv="Content-Type" content="...; charset=x">。
func charsetFromMeta(raw []byte) string {
head := raw
if len(head) > 2048 {
head = head[:2048] // 声明一定在前,不必扫全文
}
lower := bytes.ToLower(head)
i := bytes.Index(lower, metaCharset)
if i < 0 {
return ""
}
rest := head[i+len(metaCharset):]
if len(rest) == 0 {
return ""
}
// 值可能被引号包住;带引号时以配对引号收尾,不带时到引号/空格/;/ > 为止
// (不闭合的引号也当结束——http-equiv 写法里值是 content="...; charset=gbk",
// 未加引号的 charset 值正好以那个收尾引号终止)
if q := rest[0]; q == '"' || q == '\'' {
rest = rest[1:]
if end := bytes.IndexByte(rest, q); end >= 0 {
rest = rest[:end]
}
} else if end := bytes.IndexAny(rest, `"' ;>`); end >= 0 {
rest = rest[:end]
}
return normalizeCharset(string(rest))
}
func normalizeCharset(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
// HTML 标准与 IANA 的常见别名统一成 htmlindex 认得的名字
switch s {
case "gb2312", "gb_2312", "gb-2312":
return "gbk" // GBK 是 GB2312 的超集,按声明的 GB2312 解会漏字
case "x-sjis":
return "shift_jis"
case "euckr", "kr":
return "euc-kr"
case "utf8", "utf-8", "ascii", "us-ascii", "":
return s
}
return s
}
func isUTF8(name string) bool {
switch name {
case "", "utf-8", "utf8", "ascii", "us-ascii":
return true
}
return false
}
+122
View File
@@ -0,0 +1,122 @@
package linkmeta
import (
"context"
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"golang.org/x/text/encoding/simplifiedchinese"
)
// gbkPage 把 UTF-8 字符串编成 GBK——老派中文站就是这么发的。
func gbkPage(t *testing.T, s string) []byte {
t.Helper()
out, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(s))
if err != nil {
t.Fatal(err)
}
return out
}
func TestDecodeGBKMetaCharset(t *testing.T) {
// 只在 <meta> 里声明,HTTP 头不带 charset——这是最常见的写法
doc := `<html><head><meta charset="gbk"><meta property="og:title" content="围棋职业棋士的直播间"></head></html>`
raw := gbkPage(t, doc)
got := parse(decode(raw, "text/html"), mustBase(t, "https://example.com/"))
if got.Title != "围棋职业棋士的直播间" {
t.Fatalf("GBK 未正确解码: %q", got.Title)
}
}
func TestDecodeHTTPHeaderWins(t *testing.T) {
doc := `<html><head><title>标题测试</title></head></html>`
raw := gbkPage(t, doc)
got := parse(decode(raw, `text/html; charset=GB2312`), mustBase(t, "https://example.com/"))
if got.Title != "标题测试" {
t.Fatalf("HTTP 头声明的字符集未生效: %q", got.Title)
}
}
func TestDecodeHTTPEquivMeta(t *testing.T) {
doc := `<html><head><meta http-equiv="Content-Type" content="text/html; charset=gbk"><title>中文标题</title></head></html>`
got := parse(decode(gbkPage(t, doc), "text/html"), mustBase(t, "https://example.com/"))
if got.Title != "中文标题" {
t.Fatalf("http-equiv 形式未识别: %q", got.Title)
}
}
func TestDecodePassthroughAndUnknown(t *testing.T) {
utf8 := `<html><head><title>已经是 UTF-8</title></head></html>`
if got := decode([]byte(utf8), "text/html; charset=utf-8"); got != utf8 {
t.Error("UTF-8 不该走转换管线")
}
if got := decode([]byte(utf8), "text/html; charset=bogus-9999"); got != utf8 {
t.Error("认不出的字符集应退回原始字节而不是报错")
}
if got := decode([]byte("\ufeff"+utf8), "text/html"); got != utf8 {
t.Error("BOM 应被剥掉")
}
}
func TestCharsetFromHeader(t *testing.T) {
cases := map[string]string{
`text/html; charset=UTF-8`: "utf-8",
`text/html;charset="gbk"`: "gbk",
`text/html; charset=GB2312`: "gbk", // 别名归一:GBK 是超集
`text/html`: "",
}
for in, want := range cases {
if got := charsetFromHeader(in); got != want {
t.Errorf("charsetFromHeader(%q) = %q, want %q", in, got, want)
}
}
}
func TestCharsetFromMeta(t *testing.T) {
cases := map[string]string{
`<html><head><meta charset='Big5'>`: "big5",
`<html><head><meta charset=gb2312>`: "gbk",
`<meta http-equiv="Content-Type" content="text/html; charset=x-sjis">`: "shift_jis",
`<html><head><title>没有声明</title>`: "",
}
for in, want := range cases {
if got := charsetFromMeta([]byte(in)); got != want {
t.Errorf("charsetFromMeta(%q) = %q, want %q", in, got, want)
}
}
}
// 端到端:真发一个 GBK 页面(HTTP 头故意不写 charset),确认整条链路出正确卡片。
func TestFetcherDecodesGBKEndToEnd(t *testing.T) {
doc := `<html><head><meta charset="gbk"><meta property="og:title" content="鱼妹妹下棋"><meta property="og:site_name" content="某中文站"></head>`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") // 不声明 charset,逼它去嗅 meta
fmt.Fprint(w, string(gbkPage(t, doc)))
}))
defer srv.Close()
f := &Fetcher{Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, addr)
}}
c, err := f.Fetch(context.Background(), srv.URL)
if err != nil {
t.Fatal(err)
}
if c.Title != "鱼妹妹下棋" || c.Site != "某中文站" {
t.Fatalf("card=%+v", c)
}
}
func mustBase(t *testing.T, s string) *url.URL {
t.Helper()
u, err := url.Parse(s)
if err != nil {
t.Fatal(err)
}
return u
}
+186
View File
@@ -0,0 +1,186 @@
// Package linkmeta 从正文里的链接抓一份「链接卡片」素材:标题、描述、站点名、封面图。
//
// 抓取由站主写作时触发(不是读者请求触发),但目标地址仍是任意公网 URL,
// 所以按对外抓取的标准对待:只放行 http/https、限时限量限跳转,
// 并在拨号那一刻解析并拒绝内网地址——防的是「服务器自己打自己」这类 SSRF,
// 以及 DNS 先返回公网 IP、拨号时换成内网 IP 的重绑定把戏。
package linkmeta
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"oneblog/internal/model"
)
const (
// maxBody 只读文档开头就够拿到 <head>,多一个字节都不多读。
maxBody = 512 << 10
// maxRedirects 跳转链上限。
maxRedirects = 3
userAgent = "Mozilla/5.0 (compatible; ONEBot/1.0; +link-preview)"
)
// Card 是抓取结果的数据形状;定义在 model 里(要落库、要给前端),
// 这里用别名保持本包的写法。
type Card = model.LinkCard
var (
urlRe = regexp.MustCompile(`https?://[^\s<>"'\)\]]+`)
trailingCut = ".,;:!?、。)]》」》"
)
// FirstURL 取正文里第一个 http(s) 链接(Markdown 原文,含代码块里的也算,
// 站主自己不会在代码块里贴想展示的链接)。没有则空串。
func FirstURL(md string) string {
m := urlRe.FindString(md)
if m == "" {
return ""
}
return strings.TrimRight(m, trailingCut)
}
// Fetcher 抓取器。Dial 为 nil 时用带 SSRF 防护的默认拨号器;
// 测试里注入普通拨号器才能打到 httptest 的回环地址(默认会被拦掉)。
type Fetcher struct {
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
Timeout time.Duration // 0 = 默认 5s
}
// Fetch 用默认安全拨号器抓 rawURL 的元信息。ctx 控制整体时限。
func Fetch(ctx context.Context, rawURL string) (*Card, error) {
return (&Fetcher{}).Fetch(ctx, rawURL)
}
// Fetch 抓 rawURL 的元信息。
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Card, error) {
u, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil {
return nil, fmt.Errorf("linkmeta: bad url: %w", err)
}
if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("linkmeta: scheme %q not allowed", u.Scheme)
}
if u.Host == "" {
return nil, errors.New("linkmeta: empty host")
}
dial := f.Dial
if dial == nil {
dial = safeDial
}
timeout := f.Timeout
if timeout <= 0 {
timeout = 5 * time.Second
}
client := &http.Client{
Transport: &http.Transport{
DialContext: dial,
TLSHandshakeTimeout: 3 * time.Second,
// 每个跳转目标都过一遍 dial(transport 会复用),无需额外校验
ForceAttemptHTTP2: false,
},
Timeout: timeout,
CheckRedirect: limitRedirects,
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
req.Header.Set("Accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("linkmeta: fetch: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
return nil, fmt.Errorf("linkmeta: status %d", resp.StatusCode)
}
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/html") {
return nil, fmt.Errorf("linkmeta: not html (%q)", ct)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBody))
if err != nil {
return nil, fmt.Errorf("linkmeta: read: %w", err)
}
final := resp.Request.URL // 跟完跳转后的最终地址
card := parse(decode(body, resp.Header.Get("Content-Type")), final)
if card.Empty() {
return nil, errors.New("linkmeta: no usable metadata")
}
return card, nil
}
func limitRedirects(req *http.Request, via []*http.Request) error {
if len(via) > maxRedirects {
return fmt.Errorf("linkmeta: too many redirects")
}
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
return fmt.Errorf("linkmeta: redirect to %q", req.URL.Scheme)
}
return nil
}
// safeDial 解析域名后逐个筛掉内网地址,再直接拨那个 IP:
// 校验和连接之间不再重新解析,DNS 重绑定就没有窗口。
func safeDial(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
var d net.Dialer
if ip := net.ParseIP(host); ip != nil {
if blocked(ip) {
return nil, fmt.Errorf("linkmeta: %s is not a public address", ip)
}
return d.DialContext(ctx, network, addr)
}
ips, err := d.Resolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
var lastErr error
for _, ia := range ips {
if blocked(ia.IP) {
continue
}
conn, err := d.DialContext(ctx, network, net.JoinHostPort(ia.IP.String(), port))
if err == nil {
return conn, nil
}
lastErr = err
}
if lastErr != nil {
return nil, lastErr
}
return nil, fmt.Errorf("linkmeta: no public address for %s", host)
}
// blocked 报告 IP 是否属于不该由本站去连的地址段。
func blocked(ip net.IP) bool {
if ip == nil || ip.IsUnspecified() || ip.IsLoopback() || ip.IsPrivate() ||
ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() ||
ip.IsMulticast() {
return true
}
// 运营商级 NAT 与 6to4 前缀:IsPrivate 不覆盖,但同样不该出现在公网抓取里
if ip4 := ip.To4(); ip4 != nil {
return ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 // 100.64.0.0/10
}
if ip.To16() != nil {
return ip[0] == 0x20 || ip[0] == 0x3f // 2001::/32 Teredo、3ffe::/16 等保留段
}
return false
}
+191
View File
@@ -0,0 +1,191 @@
package linkmeta
import (
"context"
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
)
func TestFirstURL(t *testing.T) {
cases := map[string]string{
"看看 https://example.com/a?x=1 这个": "https://example.com/a?x=1",
"[Go 语言](https://go.dev/doc) 官方文档": "https://go.dev/doc",
"结尾标点要剪掉 https://a.cn/page。": "https://a.cn/page",
"没有链接就是空": "",
"两个 https://first.cn https://second.cn": "https://first.cn",
"裸 www.example.com 不算": "",
}
for in, want := range cases {
if got := FirstURL(in); got != want {
t.Errorf("FirstURL(%q) = %q, want %q", in, got, want)
}
}
}
func TestParsePriority(t *testing.T) {
doc := `<html><head>
<title>兜底标题</title>
<meta name="description" content="兜底描述">
<meta property="og:title" content="OG 标题">
<meta content="OG 描述" property="og:description">
<meta name="og:site_name" content="示例站">
<meta property="og:image" content="/pics/a.png">
</head><body><p>正文里的 <b>标签</b> 不该被当成标题</p></body></html>`
base, _ := url.Parse("https://example.com/post/1")
c := parse(doc, base)
if c.Title != "OG 标题" {
t.Errorf("title=%q 应优先 og:title", c.Title)
}
if c.Desc != "OG 描述" {
t.Errorf("desc=%q", c.Desc)
}
if c.Site != "示例站" {
t.Errorf("site=%q", c.Site)
}
if c.Image != "https://example.com/pics/a.png" {
t.Errorf("相对图片未补全: %q", c.Image)
}
}
func TestParseFallsBackToTitleTag(t *testing.T) {
base, _ := url.Parse("https://example.com/")
c := parse(`<html><head><title> 只有
标题 </title></head></html>`, base)
if c.Title != "只有 标题" {
t.Errorf("title=%q", c.Title)
}
if c.Site != "example.com" {
t.Errorf("site=%q 应退回主机名", c.Site)
}
}
func TestParseUnescapesEntities(t *testing.T) {
base, _ := url.Parse("https://example.com/")
c := parse(`<head><meta property="og:title" content="Tom &amp; Jerry &#34;quoted&#34;"></head>`, base)
if c.Title != `Tom & Jerry "quoted"` {
t.Errorf("title=%q", c.Title)
}
}
func TestBlockedAddresses(t *testing.T) {
want := map[string]bool{
"127.0.0.1": true,
"10.0.3.5": true,
"172.16.0.1": true,
"192.168.1.1": true,
"169.254.169.254": true, // 云元数据
"100.64.0.1": true, // CGNAT
"::1": true,
"fe80::1": true,
"fc00::1": true, // ULA
"93.184.216.34": false,
"2606:2800:220:1:248:1893:25c8:1946": false,
}
for s, wantBlocked := range want {
ip := net.ParseIP(s)
if ip == nil {
t.Fatalf("bad test ip %q", s)
}
if got := blocked(ip); got != wantBlocked {
t.Errorf("blocked(%s) = %v, want %v", s, got, wantBlocked)
}
}
}
// 默认拨号器必须拒掉回环地址——httptest 的服务就在 127.0.0.1,
// 这条同时验证了「防护生效」和「测试用的注入通道确实是必要的」。
func TestDefaultFetchRejectsLoopback(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `<html><head><title>x</title></head></html>`)
}))
defer srv.Close()
_, err := Fetch(context.Background(), srv.URL)
if err == nil {
t.Fatal("默认 Fetcher 竟然后到了回环地址,SSRF 防护形同虚设")
}
if !strings.Contains(err.Error(), "not a public address") && !strings.Contains(err.Error(), "no public address") {
t.Fatalf("err=%v,应因内网地址被拒", err)
}
}
func TestFetcherParsesRealPage(t *testing.T) {
var gotUA, gotAccept string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotUA, gotAccept = r.Header.Get("User-Agent"), r.Header.Get("Accept")
if r.URL.Path == "/nope" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprint(w, `<html><head><meta property="og:title" content="标题"><meta property="og:image" content="https://cdn.example/i.png"></head>`)
}))
defer srv.Close()
f := &Fetcher{Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, addr)
}, Timeout: 2 * time.Second}
c, err := f.Fetch(context.Background(), srv.URL+"/page")
if err != nil {
t.Fatal(err)
}
if c.Title != "标题" || c.Image != "https://cdn.example/i.png" {
t.Fatalf("card=%+v", c)
}
if !strings.HasPrefix(c.URL, srv.URL) {
t.Errorf("url=%q", c.URL)
}
if gotUA != userAgent {
t.Errorf("ua=%q", gotUA)
}
if !strings.Contains(gotAccept, "text/html") {
t.Errorf("accept=%q", gotAccept)
}
if _, err := f.Fetch(context.Background(), srv.URL+"/nope"); err == nil {
t.Error("404 应报错")
}
}
func TestFetchRejectsNonHTMLAndBadScheme(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
w.Write([]byte("binary"))
}))
defer srv.Close()
f := &Fetcher{Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, addr)
}}
if _, err := f.Fetch(context.Background(), srv.URL); err == nil {
t.Error("非 HTML 应拒绝")
}
if _, err := f.Fetch(context.Background(), "ftp://example.com/x"); err == nil {
t.Error("非 http(s) 协议应拒绝")
}
if _, err := f.Fetch(context.Background(), "not a url"); err == nil {
t.Error("坏 URL 应报错")
}
}
func TestCardEmpty(t *testing.T) {
if !(&Card{URL: "https://x.cn"}).Empty() {
t.Error("只有 URL 不算有内容")
}
if (&Card{URL: "https://x.cn", Title: "t"}).Empty() {
t.Error("有标题不该判空")
}
var nilCard *Card
if !nilCard.Empty() {
t.Error("nil 应判空")
}
}
+100
View File
@@ -0,0 +1,100 @@
package linkmeta
import (
"html"
"net/url"
"regexp"
"strings"
)
var (
headRe = regexp.MustCompile(`(?is)<head[^>]*>(.*?)</head>`)
metaRe = regexp.MustCompile(`(?is)<meta\b[^>]*?>`)
attrRe = regexp.MustCompile(`(?is)\b(property|name|http-equiv|content)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>"']+))`)
titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
tagRe = regexp.MustCompile(`(?s)<[^>]*>`)
wsRe = regexp.MustCompile(`\s+`)
)
// parse 从 HTML 里挑出卡片字段。优先 Open Graph,其次 Twitter Card,最后
// 退到 <title> 与 description——绝大多数站点至少满足其中一个。
func parse(doc string, base *url.URL) *Card {
c := &Card{URL: base.String(), Site: base.Hostname()}
head := doc
if m := headRe.FindStringSubmatch(doc); m != nil {
head = m[1]
} else if len(head) > 64<<10 {
head = head[:64<<10] // 没有闭合 <head> 时也别整篇扫
}
// 同一个 key 出现多次时先出现的赢(后面的往往是重复声明)
meta := map[string]string{}
for _, tag := range metaRe.FindAllString(head, -1) {
key, val := metaTag(tag)
if key == "" || val == "" {
continue
}
if _, seen := meta[key]; !seen {
meta[key] = val
}
}
c.Title = first(meta["og:title"], meta["twitter:title"])
if c.Title == "" {
if m := titleRe.FindStringSubmatch(head); m != nil {
c.Title = clean(m[1])
}
}
c.Desc = first(meta["og:description"], meta["twitter:description"], meta["description"])
if site := meta["og:site_name"]; site != "" {
c.Site = site
}
if img := first(meta["og:image"], meta["twitter:image"], meta["twitter:image:src"]); img != "" {
c.Image = resolve(img, base)
}
return c
}
// metaTag 从单个 <meta> 标签里取 (key, content),属性顺序不限。
func metaTag(tag string) (string, string) {
var key, val string
for _, a := range attrRe.FindAllStringSubmatch(tag, -1) {
v := first(a[3], a[4], a[5])
switch strings.ToLower(a[1]) {
case "property", "name", "http-equiv":
key = strings.ToLower(v)
case "content":
val = v
}
}
return key, clean(val)
}
// clean 去标签、解实体、压空白:抓来的文本可能带内联标签或连续换行。
func clean(s string) string {
s = html.UnescapeString(tagRe.ReplaceAllString(s, " "))
return strings.TrimSpace(wsRe.ReplaceAllString(s, " "))
}
// resolve 把相对的图片地址补成绝对 URL;解析不了就丢掉这个字段。
func resolve(ref string, base *url.URL) string {
ref = strings.TrimSpace(ref)
if ref == "" || strings.HasPrefix(ref, "data:") {
return ""
}
u, err := url.Parse(ref)
if err != nil {
return ""
}
return base.ResolveReference(u).String()
}
func first(vals ...string) string {
for _, v := range vals {
if v != "" {
return v
}
}
return ""
}
+19
View File
@@ -32,6 +32,23 @@ type Post struct {
// Images 是短文配图(快发盒上传,≤4 张,Twitter 式网格展示),
// 以 JSON 数组存在 posts.images 列里,正文 content_md 不含它们。
Images []string `json:"images"`
// LinkCard 是正文里第一个外链的预览卡片(发布时抓一次,存 posts.link_card)。
// nil = 没有链接或抓取失败,前台不渲染卡片。
LinkCard *LinkCard `json:"link_card,omitempty"`
}
// LinkCard 是链接预览卡片:由 internal/linkmeta 抓取,以 JSON 存在 posts 表里。
type LinkCard struct {
URL string `json:"url"`
Title string `json:"title,omitempty"`
Desc string `json:"desc,omitempty"`
Site string `json:"site,omitempty"`
Image string `json:"image,omitempty"`
}
// Empty 报告卡片有没有可展示的字段(只有 URL 不算)。
func (c *LinkCard) Empty() bool {
return c == nil || (c.Title == "" && c.Desc == "" && c.Image == "")
}
type PostInput struct {
@@ -47,6 +64,8 @@ type PostInput struct {
ReadingMinutes *int `json:"reading_minutes"`
// Images 是短文配图 URL(快发盒上传),nil = 不修改、空数组 = 清空
Images []string `json:"images"`
// LinkCard 由服务端在保存时按正文内容重算(站主不必发这个字段)
LinkCard *LinkCard `json:"link_card,omitempty"`
}
type Tag struct {
+46 -12
View File
@@ -135,6 +135,8 @@ func (s *Store) migrate() error {
`ALTER TABLE tags ADD COLUMN color TEXT NOT NULL DEFAULT ''`,
// 短文配图(快发盒上传,Twitter 式网格展示),JSON 数组
`ALTER TABLE posts ADD COLUMN images TEXT NOT NULL DEFAULT '[]'`,
// 正文首个外链的预览卡片(og 标题/描述/封面),JSON 对象或空串
`ALTER TABLE posts ADD COLUMN link_card TEXT NOT NULL DEFAULT ''`,
}
for _, q := range columnAdds {
if _, err := s.db.Exec(s.db.Q(q)); err != nil && !strings.Contains(err.Error(), "already exists") &&
@@ -477,14 +479,14 @@ func sanitizeOrder(o string) string {
}
const postCols = `id, kind, title, slug, summary, cover_url, content_md, content_html, status,
published_at, created_at, updated_at, reading_minutes, LENGTH(content_md), images`
published_at, created_at, updated_at, reading_minutes, LENGTH(content_md), images, link_card`
// listCols 用于列表/时间线:不传 content_md(前端不用),
// 长文 content_html 只截 600 字符供无摘要时提取纯文本,短文保留全文渲染。
const listCols = `id, kind, title, slug, summary, cover_url,
'' AS content_md,
CASE WHEN kind = 'short' THEN content_html ELSE substr(content_html, 1, 600) END AS content_html,
status, published_at, created_at, updated_at, reading_minutes, LENGTH(content_md), images`
status, published_at, created_at, updated_at, reading_minutes, LENGTH(content_md), images, link_card`
// images 列的 JSON 编解码(列存 '[]',Go 侧 []string;坏数据静默为空)。
func decodeImages(s string) []string {
@@ -509,14 +511,39 @@ func encodeImages(imgs []string) string {
return string(b)
}
// link_card 列的 JSON 编解码:空串/坏数据一律当作「没有卡片」。
func decodeLinkCard(s string) *model.LinkCard {
s = strings.TrimSpace(s)
if s == "" {
return nil
}
var c model.LinkCard
if err := json.Unmarshal([]byte(s), &c); err != nil || c.Empty() {
return nil
}
return &c
}
func encodeLinkCard(c *model.LinkCard) string {
if c.Empty() {
return ""
}
b, err := json.Marshal(c)
if err != nil {
return ""
}
return string(b)
}
func scanPost(rows interface{ Scan(...any) error }) (model.Post, error) {
var p model.Post
var imgs string
var imgs, card string
err := rows.Scan(&p.ID, &p.Kind, &p.Title, &p.Slug, &p.Summary, &p.CoverURL,
&p.ContentMd, &p.ContentHTML, &p.Status, &p.PublishedAt, &p.CreatedAt, &p.UpdatedAt,
&p.ReadingMinutes, &p.ContentLen, &imgs)
&p.ReadingMinutes, &p.ContentLen, &imgs, &card)
p.Tags = []string{}
p.Images = decodeImages(imgs)
p.LinkCard = decodeLinkCard(card)
return p, err
}
@@ -689,10 +716,10 @@ func (s *Store) GetBySlug(slug string) (model.Post, error) {
}
func scanPostInto(row *sql.Row, p *model.Post) error {
var imgs string
var imgs, card string
err := row.Scan(&p.ID, &p.Kind, &p.Title, &p.Slug, &p.Summary, &p.CoverURL,
&p.ContentMd, &p.ContentHTML, &p.Status, &p.PublishedAt, &p.CreatedAt, &p.UpdatedAt,
&p.ReadingMinutes, &p.ContentLen, &imgs)
&p.ReadingMinutes, &p.ContentLen, &imgs, &card)
if err == sql.ErrNoRows {
return ErrNotFound
}
@@ -701,6 +728,7 @@ func scanPostInto(row *sql.Row, p *model.Post) error {
}
p.Tags = []string{}
p.Images = decodeImages(imgs)
p.LinkCard = decodeLinkCard(card)
return nil
}
@@ -945,17 +973,19 @@ func (s *Store) Create(in model.PostInput) (model.Post, error) {
var id int64
q := s.db.Q(`INSERT INTO posts (kind,title,slug,summary,cover_url,content_md,content_html,status,
published_at,created_at,updated_at,reading_minutes,images)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`)
published_at,created_at,updated_at,reading_minutes,images,link_card)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
if s.db.Dialect == db.Postgres {
err := s.db.QueryRow(q, p.Kind, p.Title, p.Slug, p.Summary, p.CoverURL, p.ContentMd, p.ContentHTML,
p.Status, p.PublishedAt, p.CreatedAt, p.UpdatedAt, p.ReadingMinutes, encodeImages(in.Images)).Scan(&id)
p.Status, p.PublishedAt, p.CreatedAt, p.UpdatedAt, p.ReadingMinutes, encodeImages(in.Images),
encodeLinkCard(in.LinkCard)).Scan(&id)
if err != nil {
return p, err
}
} else {
res, err := s.db.Exec(q, p.Kind, p.Title, p.Slug, p.Summary, p.CoverURL, p.ContentMd, p.ContentHTML,
p.Status, p.PublishedAt, p.CreatedAt, p.UpdatedAt, p.ReadingMinutes, encodeImages(in.Images))
p.Status, p.PublishedAt, p.CreatedAt, p.UpdatedAt, p.ReadingMinutes, encodeImages(in.Images),
encodeLinkCard(in.LinkCard))
if err != nil {
return p, err
}
@@ -1007,6 +1037,10 @@ func (s *Store) Update(id int64, in model.PostInput) (model.Post, error) {
if in.Images != nil {
p.Images = in.Images
}
// LinkCard 由 admin 层按正文重算后传入;空卡片(含只有 URL)= 清空
if in.LinkCard != nil {
p.LinkCard = in.LinkCard
}
p.UpdatedAt = now()
if in.ReadingMinutes != nil && *in.ReadingMinutes > 0 {
p.ReadingMinutes = *in.ReadingMinutes
@@ -1015,9 +1049,9 @@ func (s *Store) Update(id int64, in model.PostInput) (model.Post, error) {
}
if _, err := s.db.Exec(s.db.Q(`UPDATE posts SET kind=?,title=?,slug=?,summary=?,cover_url=?,content_md=?,
content_html=?,status=?,published_at=?,updated_at=?,reading_minutes=?,images=? WHERE id=?`),
content_html=?,status=?,published_at=?,updated_at=?,reading_minutes=?,images=?,link_card=? WHERE id=?`),
p.Kind, p.Title, p.Slug, p.Summary, p.CoverURL, p.ContentMd, p.ContentHTML, p.Status,
p.PublishedAt, p.UpdatedAt, p.ReadingMinutes, encodeImages(p.Images), id); err != nil {
p.PublishedAt, p.UpdatedAt, p.ReadingMinutes, encodeImages(p.Images), encodeLinkCard(p.LinkCard), id); err != nil {
return p, err
}
if in.Tags != nil {