From c1c9e6025a3fc68ffdc9fdb20e6828adbcbd03e7 Mon Sep 17 00:00:00 2001 From: "henry.chen" Date: Wed, 14 Jul 2021 10:54:30 +0800 Subject: [PATCH] chore: add backup app --- cmd/backup/main.go | 50 +++++++++++++++++- cmd/eiblog/main.go | 10 ++-- conf/app.yml | 17 ++++-- pkg/config/config.go | 12 ++++- pkg/core/backup/ping/ping.go | 18 +++++++ pkg/core/backup/swag/swag.go | 15 ++++++ pkg/core/backup/timer/qiniu/qiniu.go | 77 ++++++++++++++++++++++++++++ pkg/core/backup/timer/timer.go | 68 ++++++++++++++++++++++++ pkg/core/eiblog/admin/admin.go | 18 ++++++- pkg/internal/qiniu.go | 50 ++++++++++++------ 10 files changed, 307 insertions(+), 28 deletions(-) create mode 100644 pkg/core/backup/ping/ping.go create mode 100644 pkg/core/backup/swag/swag.go create mode 100644 pkg/core/backup/timer/qiniu/qiniu.go create mode 100644 pkg/core/backup/timer/timer.go diff --git a/cmd/backup/main.go b/cmd/backup/main.go index dc684bd..5b91a44 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -1,8 +1,54 @@ // Package main provides ... package main -import "fmt" +import ( + "fmt" + + "github.com/eiblog/eiblog/pkg/config" + "github.com/eiblog/eiblog/pkg/core/backup/ping" + "github.com/eiblog/eiblog/pkg/core/backup/swag" + "github.com/eiblog/eiblog/pkg/core/backup/timer" + + "github.com/gin-gonic/gin" +) func main() { - fmt.Println("hello world!") + fmt.Println("Hi, it's App " + config.Conf.BackupApp.Name) + + endRun := make(chan error, 1) + + runTimer(endRun) + + runHTTPServer(endRun) + fmt.Println(<-endRun) +} + +func runTimer(endRun chan error) { + go func() { + endRun <- timer.Start() + }() +} + +func runHTTPServer(endRun chan error) { + if !config.Conf.EiBlogApp.EnableHTTP { + return + } + + if config.Conf.RunMode == config.ModeProd { + gin.SetMode(gin.ReleaseMode) + } + e := gin.Default() + + // swag + swag.RegisterRoutes(e) + + // route + ping.RegisterRoutes(e) + + // start + address := fmt.Sprintf(":%d", config.Conf.EiBlogApp.HTTPPort) + go func() { + endRun <- e.Run(address) + }() + fmt.Println("HTTP server running on: " + address) } diff --git a/cmd/eiblog/main.go b/cmd/eiblog/main.go index 7df894d..48a1984 100644 --- a/cmd/eiblog/main.go +++ b/cmd/eiblog/main.go @@ -19,13 +19,13 @@ import ( func main() { fmt.Println("Hi, it's App " + config.Conf.EiBlogApp.Name) - endRun := make(chan bool, 1) + endRun := make(chan error, 1) runHTTPServer(endRun) - <-endRun + fmt.Println(<-endRun) } -func runHTTPServer(endRun chan bool) { +func runHTTPServer(endRun chan error) { if !config.Conf.EiBlogApp.EnableHTTP { return } @@ -65,6 +65,8 @@ func runHTTPServer(endRun chan bool) { // start address := fmt.Sprintf(":%d", config.Conf.EiBlogApp.HTTPPort) - go e.Run(address) + go func() { + endRun <- e.Run(address) + }() fmt.Println("HTTP server running on: " + address) } diff --git a/conf/app.yml b/conf/app.yml index 5415ade..da94050 100644 --- a/conf/app.yml +++ b/conf/app.yml @@ -2,7 +2,7 @@ appname: eiblog database: driver: postgres source: host=localhost port=5432 user=postgres dbname=eiblog sslmode=disable password=MTI3LjAuMC4x -eshost: http://localhost:9200 +eshost: eiblogapp: mode: name: cmd-eiblog @@ -53,6 +53,15 @@ eiblogapp: username: deepzz # *后台登录用户名 password: deepzz # *登录明文密码 backupapp: - name: cmd-backup - enablehttp: true - httpport: 9001 + mode: + name: cmd-backup + enablehttp: true + httpport: 9001 + backupto: qiniu # 备份到七牛云 + interval: 7d # 多久备份一次 + validity: 60d # 保存时长 + qiniu: # 七牛OSS + bucket: backup + domain: st.deepzz.com + accesskey: MB6AXl_Sj_mmFsL-Lt59Dml2Vmy2o8XMmiCbbSeC + secretkey: BIrMy0fsZ0_SHNceNXk3eDuo7WmVYzj2-zrmd5Tf diff --git a/pkg/config/config.go b/pkg/config/config.go index c6cd201..b8c14a6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -121,6 +121,16 @@ type EiBlogApp struct { Blogger Blogger `yaml:"blogger"` } +// BackupApp config +type BackupApp struct { + Mode + + BackupTo string `yaml:"backupto"` + Interval string `yaml:"interval"` // circle backup, default: 7d + Validity string `yaml:"validity"` // storage days, default: 60d + Qiniu Qiniu `yaml:"qiniu"` // qiniu config +} + // Config app config type Config struct { RunMode string `yaml:"runmode"` @@ -128,7 +138,7 @@ type Config struct { Database Database `yaml:"database"` ESHost string `yaml:"eshost"` EiBlogApp EiBlogApp `yaml:"eiblogapp"` - BackupApp Mode `yaml:"backupapp"` + BackupApp BackupApp `yaml:"backupapp"` } // load config file diff --git a/pkg/core/backup/ping/ping.go b/pkg/core/backup/ping/ping.go new file mode 100644 index 0000000..f35ca3c --- /dev/null +++ b/pkg/core/backup/ping/ping.go @@ -0,0 +1,18 @@ +// Package ping provides ... +package ping + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +// RegisterRoutes register routes +func RegisterRoutes(group gin.IRoutes) { + group.GET("/ping", handlePing) +} + +// handlePing ping +func handlePing(c *gin.Context) { + c.String(http.StatusOK, "it's ok") +} diff --git a/pkg/core/backup/swag/swag.go b/pkg/core/backup/swag/swag.go new file mode 100644 index 0000000..facdaa6 --- /dev/null +++ b/pkg/core/backup/swag/swag.go @@ -0,0 +1,15 @@ +// Package swag provides ... +package swag + +import ( + _ "github.com/eiblog/eiblog/pkg/core/backup/docs" // docs + + "github.com/gin-gonic/gin" + ginSwagger "github.com/swaggo/gin-swagger" + "github.com/swaggo/gin-swagger/swaggerFiles" +) + +// RegisterRoutes register routes +func RegisterRoutes(group gin.IRoutes) { + group.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler)) +} diff --git a/pkg/core/backup/timer/qiniu/qiniu.go b/pkg/core/backup/timer/qiniu/qiniu.go new file mode 100644 index 0000000..954fd20 --- /dev/null +++ b/pkg/core/backup/timer/qiniu/qiniu.go @@ -0,0 +1,77 @@ +// Package qiniu provides ... +package qiniu + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "time" + + "github.com/eiblog/eiblog/pkg/config" + "github.com/eiblog/eiblog/pkg/internal" +) + +// Storage qiniu storage +type Storage struct{} + +// BackupData implements timer.Storage +func (s Storage) BackupData(now time.Time) error { + switch config.Conf.Database.Source { + case "mongodb": + return backupFromMongoDB(now) + default: + return errors.New("unsupported database source backup to qiniu") + } +} + +func backupFromMongoDB(now time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20) + defer cancel() + + // dump + arg := fmt.Sprintf("mongodump -h %s -d eiblog -o /tmp", + config.Conf.Database.Source) + cmd := exec.CommandContext(ctx, "sh", "-c", arg) + err := cmd.Run() + if err != nil { + return err + } + // tar + name := fmt.Sprintf("eiblog-%s.tar.gz", now.Format("2006-01-02")) + arg = fmt.Sprintf("tar czf %s /tmp/eiblog", name) + cmd = exec.CommandContext(ctx, "sh", "-c", arg) + err = cmd.Run() + if err != nil { + return err + } + + // upload file + f, err := os.Open("/tmp/eiblog/" + name) + if err != nil { + return err + } + s, err := f.Stat() + if err != nil { + return err + } + uploadParams := internal.UploadParams{ + Name: name, + Size: s.Size(), + Data: f, + + Conf: config.Conf.BackupApp.Qiniu, + } + _, err = internal.QiniuUpload(uploadParams) + if err != nil { + return err + } + // after days delete + deleteParams := internal.DeleteParams{ + Name: name, + + Conf: config.Conf.BackupApp.Qiniu, + } + return internal.QiniuDelete(deleteParams) +} diff --git a/pkg/core/backup/timer/timer.go b/pkg/core/backup/timer/timer.go new file mode 100644 index 0000000..2378a05 --- /dev/null +++ b/pkg/core/backup/timer/timer.go @@ -0,0 +1,68 @@ +// Package timer provides ... +package timer + +import ( + "errors" + "strconv" + "time" + + "github.com/eiblog/eiblog/pkg/config" + "github.com/eiblog/eiblog/pkg/core/backup/timer/qiniu" + + "github.com/sirupsen/logrus" +) + +// Start to backup with ticker +func Start() error { + var storage Storage + // backup instance + switch config.Conf.BackupApp.BackupTo { + case "qiniu": + storage = qiniu.Storage{} + + default: + return errors.New("timer: unknown backup to driver: " + + config.Conf.BackupApp.BackupTo) + } + + // parse duration + interval, err := ParseDuration(config.Conf.BackupApp.Interval) + if err != nil { + return err + } + + t := time.NewTicker(interval) + for now := range t.C { + err = storage.BackupData(now) + if err != nil { + logrus.Error("timer: Start.BackupData: ", now, err) + } + } + return nil +} + +// ParseDuration parse string to duration +func ParseDuration(d string) (time.Duration, error) { + if len(d) == 0 { + return 0, errors.New("timer: incorrect duration input") + } + + length := len(d) + switch d[length-1] { + case 's', 'm', 'h': + return time.ParseDuration(d) + case 'd': + di, err := strconv.Atoi(d[:length-1]) + if err != nil { + return 0, err + } + return time.Duration(di) * time.Hour * 24, nil + } + + return 0, errors.New("timer: unsupported duration:" + d) +} + +// Storage backup backend +type Storage interface { + BackupData(now time.Time) error +} diff --git a/pkg/core/eiblog/admin/admin.go b/pkg/core/eiblog/admin/admin.go index c688202..f1598f6 100644 --- a/pkg/core/eiblog/admin/admin.go +++ b/pkg/core/eiblog/admin/admin.go @@ -467,7 +467,15 @@ func handleAPIQiniuUpload(c *gin.Context) { return } filename := strings.ToLower(header.Filename) - url, err := internal.QiniuUpload(filename, s.Size(), file) + + params := internal.UploadParams{ + Name: filename, + Size: s.Size(), + Data: file, + + Conf: config.Conf.EiBlogApp.Qiniu, + } + url, err := internal.QiniuUpload(params) if err != nil { logrus.Error("handleAPIQiniuUpload.QiniuUpload: ", err) c.String(http.StatusBadRequest, err.Error()) @@ -491,7 +499,13 @@ func handleAPIQiniuDelete(c *gin.Context) { logrus.Error("handleAPIQiniuDelete.PostForm: 参数错误") return } - err := internal.QiniuDelete(name) + + params := internal.DeleteParams{ + Name: name, + + Conf: config.Conf.EiBlogApp.Qiniu, + } + err := internal.QiniuDelete(params) if err != nil { logrus.Error("handleAPIQiniuDelete.QiniuDelete: ", err) } diff --git a/pkg/internal/qiniu.go b/pkg/internal/qiniu.go index 202c6ad..c31e94a 100644 --- a/pkg/internal/qiniu.go +++ b/pkg/internal/qiniu.go @@ -13,19 +13,28 @@ import ( "github.com/qiniu/go-sdk/v7/storage" ) +// UploadParams upload params +type UploadParams struct { + Name string + Size int64 + Data io.Reader + + Conf config.Qiniu +} + // QiniuUpload 上传文件 -func QiniuUpload(name string, size int64, data io.Reader) (string, error) { - if config.Conf.EiBlogApp.Qiniu.AccessKey == "" || - config.Conf.EiBlogApp.Qiniu.SecretKey == "" { +func QiniuUpload(params UploadParams) (string, error) { + if params.Conf.AccessKey == "" || + params.Conf.SecretKey == "" { return "", errors.New("qiniu config error") } - key := completeQiniuKey(name) + key := completeQiniuKey(params.Name) - mac := qbox.NewMac(config.Conf.EiBlogApp.Qiniu.AccessKey, - config.Conf.EiBlogApp.Qiniu.SecretKey) + mac := qbox.NewMac(params.Conf.AccessKey, + params.Conf.SecretKey) // 设置上传策略 putPolicy := &storage.PutPolicy{ - Scope: config.Conf.EiBlogApp.Qiniu.Bucket, + Scope: params.Conf.Bucket, Expires: 3600, InsertOnly: 1, } @@ -42,20 +51,28 @@ func QiniuUpload(name string, size int64, data io.Reader) (string, error) { putExtra := &storage.PutExtra{} err := uploader.Put(context.Background(), ret, uploadToken, - key, data, size, putExtra) + key, params.Data, params.Size, putExtra) if err != nil { return "", err } - url := "https://" + config.Conf.EiBlogApp.Qiniu.Domain + "/" + key + url := "https://" + params.Conf.Domain + "/" + key return url, nil } -// QiniuDelete 删除文件 -func QiniuDelete(name string) error { - key := completeQiniuKey(name) +// DeleteParams delete params +type DeleteParams struct { + Name string + Days int - mac := qbox.NewMac(config.Conf.EiBlogApp.Qiniu.AccessKey, - config.Conf.EiBlogApp.Qiniu.SecretKey) + Conf config.Qiniu +} + +// QiniuDelete 删除文件 +func QiniuDelete(params DeleteParams) error { + key := completeQiniuKey(params.Name) + + mac := qbox.NewMac(params.Conf.AccessKey, + params.Conf.SecretKey) // 上传配置 cfg := &storage.Config{ Zone: &storage.ZoneHuadong, @@ -64,7 +81,10 @@ func QiniuDelete(name string) error { // manager bucketManager := storage.NewBucketManager(mac, cfg) // Delete - return bucketManager.Delete(config.Conf.EiBlogApp.Qiniu.Bucket, key) + if params.Days > 0 { + return bucketManager.DeleteAfterDays(params.Conf.Bucket, key, params.Days) + } + return bucketManager.Delete(params.Conf.Bucket, key) } // completeQiniuKey 修复路径