chore: add backup app

README.md

@@ -8,34 +8,52 @@
 
 ### 快速体验
 
-1、下载程序压缩包：到 [这里](https://github.com/eiblog/eiblog/releases) 下载 eiblog 相应系统压缩包，然后解压缩。
+这里以 mongodb 为例，更多支持的后端存储服务如下：
 
-2、启动数据库服务：博客支持多种数据库后端，如MongoDB、MySQL、Postgres、SQLite等。
+| 类型（driver） | 地址（source）示例                                           |
+| -------------- | ------------------------------------------------------------ |
+| mongodb        | mongodb://localhost:27017                                    |
+| mysql          | user:password@tcp(localhost:3306)/eiblog?charset=utf8mb4&parseTime=True&loc=Local |
+| postgres       | host=localhost port=5432 user=user password=password dbname=eiblog sslmode=disable |
+| sqlite         | /path/eiblog.db                                              |
+| sqlserver      | sqlserver://user:password@localhost:9930?database=eiblog     |
+| clickhouse     | tcp://localhost:9000?database=eiblog&username=user&password=password&read_timeout=10&write_timeout=20 |
+
+1、启动依赖服务，mongodb、elasticsearch：
+
+```
+$ docker run --name mongodb \
+    -p 27017:27017 \
+    -v ${PWD}/mgodb:/data/db \
+    mongo:3.2
+
+$ docker run --name elasticsearch \
+    -p 9200:9200 \
+    -p ${PWD}/esdata:/usr/share/elasticsearch/data \
+    deepzz0/elasticsearch:2.4.1
+```
+
+2、下载压缩包，到 [这里](https://github.com/eiblog/eiblog/releases) 下载 eiblog（非backup） 相应系统压缩包，然后解压缩。
+
+3、修改配置，将数据库与ES地址修改为相应地址：
 
 ```
 # 修改 conf/app.yml 数据库连接配置
-# driver可选：mongodb、mysql、postgres、sqlite、sqlserver、clickhouse、redis等
-# source为相应的连接地址
 database:
-  driver: postgres
-  source: host=localhost port=5432 user=postgres dbname=eiblog sslmode=disable password=MTI3LjAuMC4x
-```
+  driver: mongodb
+  source: mongodb://localhost:27017
 
-3、启动 ES 搜索服务：博客使用 ElasticSearch 2.4.1 做为搜索引擎。
-
-```
-# 修改 conf/app.yml ElasticSearch连接配置
-# 如果不启用搜索功能可以置空
+# 修改 conf/app.yml ES连接配置，如果不启用搜索功能可以置空
 eshost: http://localhost:9200
 ```
 
-4、启动博客程序。
+4、启动服务：
 
 ```
 ./backend
 ```
 
-然后访问 `localhost:9000` 就可以了。
+然后访问 `localhost:9000` 就可以了，后台地址 `localhost:9000/admin/login`，默认账户密码 `deepzz/deepzz`。
 
 ### 功能特性
 
@@ -47,16 +65,16 @@ eshost: http://localhost:9200
 
 功能说明：
 
-- [x] 博客归档，利用时间线帮助我们将归纳博文，内容少于一年按月归档，大于则按年归档。
-- [x] 博客专题，有时候博文是同一系列，专题能够帮助我们很好归纳博文，对阅读者是非常友好的。
-- [x] 标签系统，每篇博文都可以打上不同标签，使得在归档和专题不满足的情况下自定义归档，这块辅助搜索简直完美。
-- [x] 搜索系统，依托ElasticSearch实现的站内搜索，速度与效率并存，再加上google opensearch，搜索只流畅。
-- [x] 管理后台，内嵌全功能 `Typecho` 后台系统，全功能 `Markdown` 编辑器让你感觉什么是简洁清爽。
-- [x] 谷歌统计，由于google api的速度问题，从而实现了后端API异步统计，使得博客页面加载飞速。
-- [x] Disqus评论，国内评论系统不友好，因此选择disqus，又由于众所周知原因国内不能用，实现另类disqus评论方式。
-- [x] 多存储后端，支持mongodb、mysql、postgres、sqlite等存储后端。
-- [x] 七牛CDN，支持在 `Markdown` 编辑器直接上传附件，让你只考虑编辑内容，解放思想。
-- [x] 自动备份，支持多存储后端的备份功能，备份数据保存到七牛CDN上。
+* 博客归档，利用时间线帮助我们将归纳博文，内容少于一年按月归档，大于则按年归档。
+* 博客专题，有时候博文是同一系列，专题能够帮助我们很好归纳博文，对阅读者是非常友好的。
+* 标签系统，每篇博文都可以打上不同标签，使得在归档和专题不满足的情况下自定义归档，这块辅助搜索简直完美。
+* 搜索系统，依托ElasticSearch实现的站内搜索，速度与效率并存，再加上google opensearch，搜索只流畅。
+* 管理后台，内嵌全功能 `Typecho` 后台系统，全功能 `Markdown` 编辑器让你感觉什么是简洁清爽。
+* 谷歌统计，由于google api的速度问题，从而实现了后端API异步统计，使得博客页面加载飞速。
+* Disqus评论，国内评论系统不友好，因此选择disqus，又由于众所周知原因国内不能用，实现另类disqus评论方式。
+* 多存储后端，支持mongodb、mysql、postgres、sqlite等存储后端。
+* 七牛CDN，支持在 `Markdown` 编辑器直接上传附件，让你只考虑编辑内容，解放思想。
+* 自动备份，支持多存储后端的备份功能，备份数据保存到七牛CDN上。
 
 当然，为了让整个系统加载速度更快，还做了更多优化措施：
 

cmd/backup/main.go

@@ -1,8 +1,54 @@
 // Package main provides ...
 package main
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/eiblog/eiblog/pkg/config"
+	"github.com/eiblog/eiblog/pkg/core/backup/ping"
+	"github.com/eiblog/eiblog/pkg/core/backup/swag"
+	"github.com/eiblog/eiblog/pkg/core/backup/timer"
+
+	"github.com/gin-gonic/gin"
+)
 
 func main() {
-	fmt.Println("hello world!")
+	fmt.Println("Hi, it's App " + config.Conf.BackupApp.Name)
+
+	endRun := make(chan error, 1)
+
+	runTimer(endRun)
+
+	runHTTPServer(endRun)
+	fmt.Println(<-endRun)
+}
+
+func runTimer(endRun chan error) {
+	go func() {
+		endRun <- timer.Start()
+	}()
+}
+
+func runHTTPServer(endRun chan error) {
+	if !config.Conf.EiBlogApp.EnableHTTP {
+		return
+	}
+
+	if config.Conf.RunMode == config.ModeProd {
+		gin.SetMode(gin.ReleaseMode)
+	}
+	e := gin.Default()
+
+	// swag
+	swag.RegisterRoutes(e)
+
+	// route
+	ping.RegisterRoutes(e)
+
+	// start
+	address := fmt.Sprintf(":%d", config.Conf.EiBlogApp.HTTPPort)
+	go func() {
+		endRun <- e.Run(address)
+	}()
+	fmt.Println("HTTP server running on: " + address)
 }

cmd/eiblog/main.go

@@ -19,13 +19,13 @@ import (
 func main() {
 	fmt.Println("Hi, it's App " + config.Conf.EiBlogApp.Name)
 
-	endRun := make(chan bool, 1)
+	endRun := make(chan error, 1)
 
 	runHTTPServer(endRun)
-	<-endRun
+	fmt.Println(<-endRun)
 }
 
-func runHTTPServer(endRun chan bool) {
+func runHTTPServer(endRun chan error) {
 	if !config.Conf.EiBlogApp.EnableHTTP {
 		return
 	}
@@ -65,6 +65,8 @@ func runHTTPServer(endRun chan bool) {
 
 	// start
 	address := fmt.Sprintf(":%d", config.Conf.EiBlogApp.HTTPPort)
-	go e.Run(address)
+	go func() {
+		endRun <- e.Run(address)
+	}()
 	fmt.Println("HTTP server running on: " + address)
 }

conf/app.yml

@@ -2,7 +2,7 @@ appname: eiblog
 database:
   driver: postgres
   source: host=localhost port=5432 user=postgres dbname=eiblog sslmode=disable password=MTI3LjAuMC4x
-eshost: http://localhost:9200
+eshost:
 eiblogapp:
   mode:
     name: cmd-eiblog
@@ -53,6 +53,15 @@ eiblogapp:
     username: deepzz # *后台登录用户名
     password: deepzz # *登录明文密码
 backupapp:
-  name: cmd-backup
-  enablehttp: true
-  httpport: 9001
+  mode:
+    name: cmd-backup
+    enablehttp: true
+    httpport: 9001
+  backupto: qiniu # 备份到七牛云
+  interval: 7d # 多久备份一次
+  validity: 60d # 保存时长
+  qiniu: # 七牛OSS
+    bucket: backup
+    domain: st.deepzz.com
+    accesskey: MB6AXl_Sj_mmFsL-Lt59Dml2Vmy2o8XMmiCbbSeC
+    secretkey: BIrMy0fsZ0_SHNceNXk3eDuo7WmVYzj2-zrmd5Tf

examples/README.md

@@ -1 +0,0 @@
-Examples for your applications and/or public libraries.

init/README.md

@@ -1 +0,0 @@
-System init (systemd, upstart, sysv) and process manager/supervisor (runit, supervisord) configs.

pkg/config/config.go

@@ -121,6 +121,16 @@ type EiBlogApp struct {
 	Blogger       Blogger  `yaml:"blogger"`
 }
 
+// BackupApp config
+type BackupApp struct {
+	Mode
+
+	BackupTo string `yaml:"backupto"`
+	Interval string `yaml:"interval"` // circle backup, default: 7d
+	Validity string `yaml:"validity"` // storage days, default: 60d
+	Qiniu    Qiniu  `yaml:"qiniu"`    // qiniu config
+}
+
 // Config app config
 type Config struct {
 	RunMode   string    `yaml:"runmode"`
@@ -128,7 +138,7 @@ type Config struct {
 	Database  Database  `yaml:"database"`
 	ESHost    string    `yaml:"eshost"`
 	EiBlogApp EiBlogApp `yaml:"eiblogapp"`
-	BackupApp Mode      `yaml:"backupapp"`
+	BackupApp BackupApp `yaml:"backupapp"`
 }
 
 // load config file

pkg/core/backup/ping/ping.go

@@ -0,0 +1,18 @@
+// Package ping provides ...
+package ping
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+)
+
+// RegisterRoutes register routes
+func RegisterRoutes(group gin.IRoutes) {
+	group.GET("/ping", handlePing)
+}
+
+// handlePing ping
+func handlePing(c *gin.Context) {
+	c.String(http.StatusOK, "it's ok")
+}

pkg/core/backup/swag/swag.go

@@ -0,0 +1,15 @@
+// Package swag provides ...
+package swag
+
+import (
+	_ "github.com/eiblog/eiblog/pkg/core/backup/docs" // docs
+
+	"github.com/gin-gonic/gin"
+	ginSwagger "github.com/swaggo/gin-swagger"
+	"github.com/swaggo/gin-swagger/swaggerFiles"
+)
+
+// RegisterRoutes register routes
+func RegisterRoutes(group gin.IRoutes) {
+	group.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))
+}

pkg/core/backup/timer/qiniu/qiniu.go

@@ -0,0 +1,77 @@
+// Package qiniu provides ...
+package qiniu
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/eiblog/eiblog/pkg/config"
+	"github.com/eiblog/eiblog/pkg/internal"
+)
+
+// Storage qiniu storage
+type Storage struct{}
+
+// BackupData implements timer.Storage
+func (s Storage) BackupData(now time.Time) error {
+	switch config.Conf.Database.Source {
+	case "mongodb":
+		return backupFromMongoDB(now)
+	default:
+		return errors.New("unsupported database source backup to qiniu")
+	}
+}
+
+func backupFromMongoDB(now time.Time) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20)
+	defer cancel()
+
+	// dump
+	arg := fmt.Sprintf("mongodump -h %s -d eiblog -o /tmp",
+		config.Conf.Database.Source)
+	cmd := exec.CommandContext(ctx, "sh", "-c", arg)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	// tar
+	name := fmt.Sprintf("eiblog-%s.tar.gz", now.Format("2006-01-02"))
+	arg = fmt.Sprintf("tar czf %s /tmp/eiblog", name)
+	cmd = exec.CommandContext(ctx, "sh", "-c", arg)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	// upload file
+	f, err := os.Open("/tmp/eiblog/" + name)
+	if err != nil {
+		return err
+	}
+	s, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	uploadParams := internal.UploadParams{
+		Name: name,
+		Size: s.Size(),
+		Data: f,
+
+		Conf: config.Conf.BackupApp.Qiniu,
+	}
+	_, err = internal.QiniuUpload(uploadParams)
+	if err != nil {
+		return err
+	}
+	// after days delete
+	deleteParams := internal.DeleteParams{
+		Name: name,
+
+		Conf: config.Conf.BackupApp.Qiniu,
+	}
+	return internal.QiniuDelete(deleteParams)
+}

pkg/core/backup/timer/timer.go

@@ -0,0 +1,68 @@
+// Package timer provides ...
+package timer
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/eiblog/eiblog/pkg/config"
+	"github.com/eiblog/eiblog/pkg/core/backup/timer/qiniu"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Start to backup with ticker
+func Start() error {
+	var storage Storage
+	// backup instance
+	switch config.Conf.BackupApp.BackupTo {
+	case "qiniu":
+		storage = qiniu.Storage{}
+
+	default:
+		return errors.New("timer: unknown backup to driver: " +
+			config.Conf.BackupApp.BackupTo)
+	}
+
+	// parse duration
+	interval, err := ParseDuration(config.Conf.BackupApp.Interval)
+	if err != nil {
+		return err
+	}
+
+	t := time.NewTicker(interval)
+	for now := range t.C {
+		err = storage.BackupData(now)
+		if err != nil {
+			logrus.Error("timer: Start.BackupData: ", now, err)
+		}
+	}
+	return nil
+}
+
+// ParseDuration parse string to duration
+func ParseDuration(d string) (time.Duration, error) {
+	if len(d) == 0 {
+		return 0, errors.New("timer: incorrect duration input")
+	}
+
+	length := len(d)
+	switch d[length-1] {
+	case 's', 'm', 'h':
+		return time.ParseDuration(d)
+	case 'd':
+		di, err := strconv.Atoi(d[:length-1])
+		if err != nil {
+			return 0, err
+		}
+		return time.Duration(di) * time.Hour * 24, nil
+	}
+
+	return 0, errors.New("timer: unsupported duration:" + d)
+}
+
+// Storage backup backend
+type Storage interface {
+	BackupData(now time.Time) error
+}

pkg/core/eiblog/admin/admin.go

@@ -467,7 +467,15 @@ func handleAPIQiniuUpload(c *gin.Context) {
 		return
 	}
 	filename := strings.ToLower(header.Filename)
-	url, err := internal.QiniuUpload(filename, s.Size(), file)
+
+	params := internal.UploadParams{
+		Name: filename,
+		Size: s.Size(),
+		Data: file,
+
+		Conf: config.Conf.EiBlogApp.Qiniu,
+	}
+	url, err := internal.QiniuUpload(params)
 	if err != nil {
 		logrus.Error("handleAPIQiniuUpload.QiniuUpload: ", err)
 		c.String(http.StatusBadRequest, err.Error())
@@ -491,7 +499,13 @@ func handleAPIQiniuDelete(c *gin.Context) {
 		logrus.Error("handleAPIQiniuDelete.PostForm: 参数错误")
 		return
 	}
-	err := internal.QiniuDelete(name)
+
+	params := internal.DeleteParams{
+		Name: name,
+
+		Conf: config.Conf.EiBlogApp.Qiniu,
+	}
+	err := internal.QiniuDelete(params)
 	if err != nil {
 		logrus.Error("handleAPIQiniuDelete.QiniuDelete: ", err)
 	}

pkg/internal/qiniu.go

@@ -13,19 +13,28 @@ import (
 	"github.com/qiniu/go-sdk/v7/storage"
 )
 
+// UploadParams upload params
+type UploadParams struct {
+	Name string
+	Size int64
+	Data io.Reader
+
+	Conf config.Qiniu
+}
+
 // QiniuUpload 上传文件
-func QiniuUpload(name string, size int64, data io.Reader) (string, error) {
-	if config.Conf.EiBlogApp.Qiniu.AccessKey == "" ||
-		config.Conf.EiBlogApp.Qiniu.SecretKey == "" {
+func QiniuUpload(params UploadParams) (string, error) {
+	if params.Conf.AccessKey == "" ||
+		params.Conf.SecretKey == "" {
 		return "", errors.New("qiniu config error")
 	}
-	key := completeQiniuKey(name)
+	key := completeQiniuKey(params.Name)
 
-	mac := qbox.NewMac(config.Conf.EiBlogApp.Qiniu.AccessKey,
-		config.Conf.EiBlogApp.Qiniu.SecretKey)
+	mac := qbox.NewMac(params.Conf.AccessKey,
+		params.Conf.SecretKey)
 	// 设置上传策略
 	putPolicy := &storage.PutPolicy{
-		Scope:      config.Conf.EiBlogApp.Qiniu.Bucket,
+		Scope:      params.Conf.Bucket,
 		Expires:    3600,
 		InsertOnly: 1,
 	}
@@ -42,20 +51,28 @@ func QiniuUpload(name string, size int64, data io.Reader) (string, error) {
 	putExtra := &storage.PutExtra{}
 
 	err := uploader.Put(context.Background(), ret, uploadToken,
-		key, data, size, putExtra)
+		key, params.Data, params.Size, putExtra)
 	if err != nil {
 		return "", err
 	}
-	url := "https://" + config.Conf.EiBlogApp.Qiniu.Domain + "/" + key
+	url := "https://" + params.Conf.Domain + "/" + key
 	return url, nil
 }
 
-// QiniuDelete 删除文件
-func QiniuDelete(name string) error {
-	key := completeQiniuKey(name)
+// DeleteParams delete params
+type DeleteParams struct {
+	Name string
+	Days int
 
-	mac := qbox.NewMac(config.Conf.EiBlogApp.Qiniu.AccessKey,
-		config.Conf.EiBlogApp.Qiniu.SecretKey)
+	Conf config.Qiniu
+}
+
+// QiniuDelete 删除文件
+func QiniuDelete(params DeleteParams) error {
+	key := completeQiniuKey(params.Name)
+
+	mac := qbox.NewMac(params.Conf.AccessKey,
+		params.Conf.SecretKey)
 	// 上传配置
 	cfg := &storage.Config{
 		Zone:     &storage.ZoneHuadong,
@@ -64,7 +81,10 @@ func QiniuDelete(name string) error {
 	// manager
 	bucketManager := storage.NewBucketManager(mac, cfg)
 	// Delete
-	return bucketManager.Delete(config.Conf.EiBlogApp.Qiniu.Bucket, key)
+	if params.Days > 0 {
+		return bucketManager.DeleteAfterDays(params.Conf.Bucket, key, params.Days)
+	}
+	return bucketManager.Delete(params.Conf.Bucket, key)
 }
 
 // completeQiniuKey 修复路径

third/README.md

@@ -1 +0,0 @@
-External helper tools, forked code and other 3rd party utilities (e.g., Swagger UI).