107 lines
3.0 KiB
Go
107 lines
3.0 KiB
Go
package linkmeta
|
|
|
|
import (
|
|
"html"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
var (
|
|
headRe = regexp.MustCompile(`(?is)<head[^>]*>(.*?)</head>`)
|
|
metaRe = regexp.MustCompile(`(?is)<meta\b[^>]*?>`)
|
|
attrRe = regexp.MustCompile(`(?is)\b(property|name|http-equiv|content)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>"']+))`)
|
|
titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
|
|
tagRe = regexp.MustCompile(`(?s)<[^>]*>`)
|
|
wsRe = regexp.MustCompile(`\s+`)
|
|
)
|
|
|
|
// parse 从 HTML 里挑出卡片字段。优先 Open Graph，其次 Twitter Card，最后
|
|
// 退到 <title> 与 description——绝大多数站点至少满足其中一个。
|
|
func parse(doc string, base *url.URL) *Card {
|
|
c := &Card{URL: base.String(), Site: base.Hostname()}
|
|
|
|
head := doc
|
|
if m := headRe.FindStringSubmatch(doc); m != nil {
|
|
head = m[1]
|
|
} else if len(head) > 64<<10 {
|
|
head = head[:64<<10] // 没有闭合 <head> 时也别整篇扫
|
|
}
|
|
|
|
// 同一个 key 出现多次时先出现的赢（后面的往往是重复声明）
|
|
meta := map[string]string{}
|
|
for _, tag := range metaRe.FindAllString(head, -1) {
|
|
key, val := metaTag(tag)
|
|
if key == "" || val == "" {
|
|
continue
|
|
}
|
|
if _, seen := meta[key]; !seen {
|
|
meta[key] = val
|
|
}
|
|
}
|
|
|
|
c.Title = first(meta["og:title"], meta["twitter:title"])
|
|
if c.Title == "" {
|
|
if m := titleRe.FindStringSubmatch(head); m != nil {
|
|
c.Title = clean(m[1])
|
|
}
|
|
}
|
|
c.Desc = first(meta["og:description"], meta["twitter:description"], meta["description"])
|
|
if site := meta["og:site_name"]; site != "" {
|
|
c.Site = site
|
|
}
|
|
if img := first(meta["og:image"], meta["twitter:image"], meta["twitter:image:src"]); img != "" {
|
|
c.Image = resolve(img, base)
|
|
}
|
|
// X（Twitter）对非白名单爬虫只回一张全站通用灰图占位符，
|
|
// 拿它当封面只会显示一张无意义的底图——当作没有图处理。
|
|
// 真推文图只发给 Twitterbot 等白名单 UA，服务端拿不到，不硬来。
|
|
if strings.Contains(c.Image, "abs.twimg.com/rweb/ssr/default") {
|
|
c.Image = ""
|
|
}
|
|
return c
|
|
}
|
|
|
|
// metaTag 从单个 <meta> 标签里取 (key, content)，属性顺序不限。
|
|
func metaTag(tag string) (string, string) {
|
|
var key, val string
|
|
for _, a := range attrRe.FindAllStringSubmatch(tag, -1) {
|
|
v := first(a[3], a[4], a[5])
|
|
switch strings.ToLower(a[1]) {
|
|
case "property", "name", "http-equiv":
|
|
key = strings.ToLower(v)
|
|
case "content":
|
|
val = v
|
|
}
|
|
}
|
|
return key, clean(val)
|
|
}
|
|
|
|
// clean 去标签、解实体、压空白：抓来的文本可能带内联标签或连续换行。
|
|
func clean(s string) string {
|
|
s = html.UnescapeString(tagRe.ReplaceAllString(s, " "))
|
|
return strings.TrimSpace(wsRe.ReplaceAllString(s, " "))
|
|
}
|
|
|
|
// resolve 把相对的图片地址补成绝对 URL；解析不了就丢掉这个字段。
|
|
func resolve(ref string, base *url.URL) string {
|
|
ref = strings.TrimSpace(ref)
|
|
if ref == "" || strings.HasPrefix(ref, "data:") {
|
|
return ""
|
|
}
|
|
u, err := url.Parse(ref)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return base.ResolveReference(u).String()
|
|
}
|
|
|
|
func first(vals ...string) string {
|
|
for _, v := range vals {
|
|
if v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|