编辑器三件套(参考 xLog):CodeMirror 源码模式 + 外链图片一键转存 + 上传进度/前置校验
- md 页签从裸 textarea 升级 CodeMirror 6:markdown 语法高亮、行内历史,粘贴/拖图直传; markdown 仍是唯一真相源,转存/切换用 syncCM 全量灌回,工具栏选区辅助改为面向 CM - POST /api/admin/files/import:外链抓取复用 linkmeta SSRF 防护拨号与跳转限制, 二进制传输放宽 30s;类型按内容嗅探反查白名单,与手动上传共用去重/落盘(persistFile 抽取共享) - 上传走 XHR 进度回调 + 编辑器 toast 栈:类型/大小前置校验(与后端白名单一致)、逐文件进度 - persistFile 去重路径补 URL 解析——此前命中去重返回的行没有 URL,转存替换会把正文图链清空(已修复受损数据) - hub nil 安全(测试环境未装配时不 panic)
This commit is contained in:
@@ -13,7 +13,9 @@ import (
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -23,6 +25,7 @@ import (
|
||||
"oneblog/internal/config"
|
||||
"oneblog/internal/httpx"
|
||||
"oneblog/internal/hub"
|
||||
"oneblog/internal/linkmeta"
|
||||
"oneblog/internal/model"
|
||||
"oneblog/internal/storage"
|
||||
"oneblog/internal/store"
|
||||
@@ -79,6 +82,7 @@ func (a *API) Routes() http.Handler {
|
||||
mux.HandleFunc("/api/admin/projects", a.guard(a.listProjects))
|
||||
mux.HandleFunc("/api/admin/projects/", a.guard(a.projectByID))
|
||||
mux.HandleFunc("/api/admin/files", a.guard(a.files))
|
||||
mux.HandleFunc("/api/admin/files/import", a.guard(a.importFiles))
|
||||
mux.HandleFunc("/api/admin/files/", a.guard(a.fileByID))
|
||||
mux.HandleFunc("/api/admin/settings", a.guard(a.settings))
|
||||
mux.HandleFunc("/api/admin/comments", a.guard(a.adminComments))
|
||||
@@ -727,6 +731,12 @@ func (a *API) storeOne(ctx context.Context, fh *multipart.FileHeader) (model.Fil
|
||||
}
|
||||
|
||||
sum := hex.EncodeToString(hasher.Sum(nil))
|
||||
return a.persistFile(ctx, fh.Filename, ext, mime, tmp.Name(), size, sum)
|
||||
}
|
||||
|
||||
// persistFile 落库共享段:内容哈希做 key(同年月分目录)、去重复用、
|
||||
// Put 对象存储、建行。storeOne(本地上传)与 importOne(外链转存)共用。
|
||||
func (a *API) persistFile(ctx context.Context, name, ext, mime, tmpPath string, size int64, sum string) (model.File, error) {
|
||||
key := fmt.Sprintf("%s/%s%s", time.Now().UTC().Format("2006/01"), sum[:12], ext)
|
||||
// S3Api 端点带路径段时(如 .../oss),该段会折进对象 key——
|
||||
// 数据库必须记录同样的完整 key,直链才不会 404
|
||||
@@ -734,12 +744,15 @@ func (a *API) storeOne(ctx context.Context, fh *multipart.FileHeader) (model.Fil
|
||||
key = p + "/" + key
|
||||
}
|
||||
|
||||
// 内容去重:同一份内容只存一份,复用已有行
|
||||
// 内容去重:同一份内容只存一份,复用已有行。
|
||||
// URL 必须按当前存储配置重新解析——去重路径不走下面的 created 赋值,
|
||||
// 漏了它转存替换会拿到空 URL(真实事故:正文图片链接被清空)。
|
||||
if exist, err := a.Store.GetFileByKey(key); err == nil {
|
||||
exist.URL = storage.FileURL(exist.Store, exist.Key, a.Cfg.UploadsPublicBase)
|
||||
return exist, nil
|
||||
}
|
||||
|
||||
f, err := os.Open(tmp.Name())
|
||||
f, err := os.Open(tmpPath)
|
||||
if err != nil {
|
||||
return model.File{}, err
|
||||
}
|
||||
@@ -750,7 +763,7 @@ func (a *API) storeOne(ctx context.Context, fh *multipart.FileHeader) (model.Fil
|
||||
|
||||
created, err := a.Store.CreateFile(model.File{
|
||||
Key: key,
|
||||
Name: fh.Filename,
|
||||
Name: name,
|
||||
Mime: mime,
|
||||
Size: size,
|
||||
SHA256: sum,
|
||||
@@ -763,6 +776,99 @@ func (a *API) storeOne(ctx context.Context, fh *multipart.FileHeader) (model.Fil
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// mimeToExt 是 allowFileExt 的反向映射:外链转存时内容嗅探出 mime,
|
||||
// 反推扩展名(URL 本身可能不带后缀或后缀不可信)。
|
||||
var mimeToExt = func() map[string]string {
|
||||
m := make(map[string]string, len(allowFileExt))
|
||||
for ext, mime := range allowFileExt {
|
||||
m[mime] = ext
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
||||
// importFiles 外链转存:POST /api/admin/files/import {"urls": [...]}。
|
||||
// 站主把别处的图片贴进正文后一键搬进自己的存储——与手动上传同一套
|
||||
// 白名单、内容嗅探与内容去重;抓取走 linkmeta 的 SSRF 防护拨号。
|
||||
func (a *API) importFiles(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
httpx.Error(w, http.StatusMethodNotAllowed, "POST required")
|
||||
return
|
||||
}
|
||||
var in struct {
|
||||
URLs []string `json:"urls"`
|
||||
}
|
||||
if err := httpx.Decode(r, &in); err != nil {
|
||||
httpx.BadRequest(w, "invalid body")
|
||||
return
|
||||
}
|
||||
if len(in.URLs) == 0 || len(in.URLs) > 20 {
|
||||
httpx.BadRequest(w, "urls 需要1-20个")
|
||||
return
|
||||
}
|
||||
// files 带上 source(原址),前端按它做正文替换
|
||||
type imported struct {
|
||||
Source string `json:"source"`
|
||||
File model.File `json:"file"`
|
||||
}
|
||||
files := make([]imported, 0, len(in.URLs))
|
||||
errs := map[string]string{}
|
||||
for _, u := range in.URLs {
|
||||
f, err := a.importOne(r.Context(), u)
|
||||
if err != nil {
|
||||
if bu, ok := err.(badUpload); ok {
|
||||
errs[u] = string(bu)
|
||||
} else {
|
||||
errs[u] = err.Error()
|
||||
}
|
||||
continue
|
||||
}
|
||||
files = append(files, imported{Source: u, File: f})
|
||||
}
|
||||
httpx.OK(w, map[string]any{"files": files, "errors": errs})
|
||||
}
|
||||
|
||||
func (a *API) importOne(ctx context.Context, rawURL string) (model.File, error) {
|
||||
cctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
data, ct, err := linkmeta.FetchBytes(cctx, rawURL, maxFileUpload)
|
||||
if err != nil {
|
||||
return model.File{}, fmt.Errorf("抓取失败:%w", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return model.File{}, badUpload("空内容")
|
||||
}
|
||||
// 类型必须落在本站白名单里:嗅探优先(不信响应头,更不信 URL 后缀)
|
||||
detected := strings.SplitN(http.DetectContentType(data[:512]), ";", 2)[0]
|
||||
ext, ok := mimeToExt[detected]
|
||||
if !ok {
|
||||
return model.File{}, badUpload("不支持的类型 " + detected)
|
||||
}
|
||||
_ = ct
|
||||
// 落临时文件:persistFile / S3 PutObject 都要确定的文件与长度
|
||||
tmp, err := os.CreateTemp("", "one-import-*")
|
||||
if err != nil {
|
||||
return model.File{}, err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
size, err := tmp.Write(data)
|
||||
if err != nil {
|
||||
tmp.Close()
|
||||
return model.File{}, err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return model.File{}, err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
// 名字取 URL 路径末段(仅用于文件管理页展示,不参与存储路径)
|
||||
name := rawURL
|
||||
if u, err := url.Parse(rawURL); err == nil && u.Path != "" {
|
||||
if base := path.Base(u.Path); base != "" && base != "/" && base != "." {
|
||||
name = base
|
||||
}
|
||||
}
|
||||
return a.persistFile(ctx, name, ext, detected, tmp.Name(), int64(size), hex.EncodeToString(sum[:]))
|
||||
}
|
||||
|
||||
func (a *API) fileByID(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := parseInt(strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/admin/files/"), "/"))
|
||||
if err != nil {
|
||||
|
||||
@@ -18,6 +18,11 @@ func New() *Hub {
|
||||
// Subscribe 订阅某主题;返回信号 channel 和退订函数。
|
||||
// channel 容量 1:订阅者处理不过来时新信号直接丢弃(合并刷新)。
|
||||
func (h *Hub) Subscribe(topic int64) (<-chan struct{}, func()) {
|
||||
// nil hub(测试/未装配场景)静默降级:永远收不到信号的通道
|
||||
if h == nil {
|
||||
ch := make(chan struct{})
|
||||
return ch, func() {}
|
||||
}
|
||||
ch := make(chan struct{}, 1)
|
||||
h.mu.Lock()
|
||||
if h.subs[topic] == nil {
|
||||
@@ -38,6 +43,9 @@ func (h *Hub) Subscribe(topic int64) (<-chan struct{}, func()) {
|
||||
|
||||
// Broadcast 唤醒某主题的全部订阅者;积压的订阅者不阻塞。
|
||||
func (h *Hub) Broadcast(topic int64) {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for ch := range h.subs[topic] {
|
||||
|
||||
@@ -75,25 +75,7 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Card, error) {
|
||||
return nil, errors.New("linkmeta: empty host")
|
||||
}
|
||||
|
||||
dial := f.Dial
|
||||
if dial == nil {
|
||||
dial = safeDial
|
||||
}
|
||||
timeout := f.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: dial,
|
||||
TLSHandshakeTimeout: 3 * time.Second,
|
||||
// 每个跳转目标都过一遍 dial(transport 会复用),无需额外校验
|
||||
ForceAttemptHTTP2: false,
|
||||
},
|
||||
Timeout: timeout,
|
||||
CheckRedirect: limitRedirects,
|
||||
}
|
||||
|
||||
client := f.client()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -125,6 +107,78 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Card, error) {
|
||||
return card, nil
|
||||
}
|
||||
|
||||
// client 组装带 SSRF 防护的 HTTP 客户端(Fetch 与 FetchBytes 共用)
|
||||
func (f *Fetcher) client() *http.Client {
|
||||
dial := f.Dial
|
||||
if dial == nil {
|
||||
dial = safeDial
|
||||
}
|
||||
timeout := f.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
return &http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: dial,
|
||||
TLSHandshakeTimeout: 3 * time.Second,
|
||||
// 每个跳转目标都过一遍 dial(transport 会复用),无需额外校验
|
||||
ForceAttemptHTTP2: false,
|
||||
},
|
||||
Timeout: timeout,
|
||||
CheckRedirect: limitRedirects,
|
||||
}
|
||||
}
|
||||
|
||||
// FetchBytes 抓二进制内容(外链图片转存用):与 Fetch 共用同一套
|
||||
// SSRF 防护与跳转限制,字节数有 maxBytes 硬上限。
|
||||
// 返回内容与 Content-Type(响应头缺失时用内容嗅探兜底)。
|
||||
func FetchBytes(ctx context.Context, rawURL string, maxBytes int64) ([]byte, string, error) {
|
||||
return (&Fetcher{}).fetchBytes(ctx, rawURL, maxBytes)
|
||||
}
|
||||
|
||||
func (f *Fetcher) fetchBytes(ctx context.Context, rawURL string, maxBytes int64) ([]byte, string, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("linkmeta: bad url: %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, "", fmt.Errorf("linkmeta: scheme %q not allowed", u.Scheme)
|
||||
}
|
||||
if u.Host == "" {
|
||||
return nil, "", errors.New("linkmeta: empty host")
|
||||
}
|
||||
client := f.client()
|
||||
// 二进制传输放宽时限:大图慢链路 5 秒的元信息默认值不够用
|
||||
if f.Timeout <= 0 {
|
||||
client.Timeout = 30 * time.Second
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("linkmeta: fetch: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
return nil, "", fmt.Errorf("linkmeta: status %d", resp.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("linkmeta: read: %w", err)
|
||||
}
|
||||
if int64(len(data)) > maxBytes {
|
||||
return nil, "", fmt.Errorf("linkmeta: exceeds %d bytes", maxBytes)
|
||||
}
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
if ct == "" {
|
||||
ct = http.DetectContentType(data)
|
||||
}
|
||||
return data, ct, nil
|
||||
}
|
||||
|
||||
func limitRedirects(req *http.Request, via []*http.Request) error {
|
||||
if len(via) > maxRedirects {
|
||||
return fmt.Errorf("linkmeta: too many redirects")
|
||||
|
||||
Reference in New Issue
Block a user