feat(backup): add restore flag

cmd/backup/main.go

@@ -2,6 +2,7 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 
 	"github.com/eiblog/eiblog/pkg/config"
@@ -12,20 +13,27 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+var restore bool
+
+func init() {
+	flag.BoolVar(&restore, "restore", false, "restore data into mongodb")
+}
+
 func main() {
 	fmt.Println("Hi, it's App " + config.Conf.BackupApp.Name)
+	flag.Parse()
 
 	endRun := make(chan error, 1)
 
-	runTimer(endRun)
+	runCommand(restore, endRun)
 
 	runHTTPServer(endRun)
 	fmt.Println(<-endRun)
 }
 
-func runTimer(endRun chan error) {
+func runCommand(restore bool, endRun chan error) {
 	go func() {
-		endRun <- timer.Start()
+		endRun <- timer.Start(restore)
 	}()
 }
 

pkg/core/backup/timer/qiniu/qiniu.go

@@ -27,6 +27,16 @@ func (s Storage) BackupData(now time.Time) error {
 	}
 }
 
+// RestoreData implements timer.Storage
+func (s Storage) RestoreData() error {
+	switch config.Conf.Database.Driver {
+	case "mongodb":
+		return restoreToMongoDB()
+	default:
+		return errors.New("unsupported database source backup to qiniu")
+	}
+}
+
 func backupFromMongoDB(now time.Time) error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20)
 	defer cancel()
@@ -61,9 +71,10 @@ func backupFromMongoDB(now time.Time) error {
 		return err
 	}
 	uploadParams := internal.UploadParams{
-		Name: name,
+		Name:           name,
-		Size: s.Size(),
+		Size:           s.Size(),
-		Data: f,
+		Data:           f,
+		NoCompletePath: true,
 
 		Conf: config.Conf.BackupApp.Qiniu,
 	}
@@ -73,10 +84,43 @@ func backupFromMongoDB(now time.Time) error {
 	}
 	// after days delete
 	deleteParams := internal.DeleteParams{
-		Name: name,
+		Name:           name,
-		Days: config.Conf.BackupApp.Validity,
+		Days:           config.Conf.BackupApp.Validity,
+		NoCompletePath: true,
 
 		Conf: config.Conf.BackupApp.Qiniu,
 	}
 	return internal.QiniuDelete(deleteParams)
 }
+
+func restoreToMongoDB() error {
+	params := internal.ContentParams{
+		Prefix: "eiblog",
+
+		Conf: config.Conf.BackupApp.Qiniu,
+	}
+	raw, err := internal.QiniuContent(params)
+	if err != nil {
+		return err
+	}
+	f, err := os.OpenFile("/tmp/eiblog.tar.gz", os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return err
+	}
+	_, _ = f.Write(raw)
+	defer f.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20)
+	defer cancel()
+	// unarchive
+	arg := fmt.Sprintf("tar xzf /tmp/eiblog.tar.gz -C /tmp")
+	cmd := exec.CommandContext(ctx, "sh", "-c", arg)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	// restore
+	arg = fmt.Sprintf("mongorestore -h %s -d eiblog /tmp/eiblog", config.Conf.Database.Source)
+	cmd = exec.CommandContext(ctx, "sh", "-c", arg)
+	return cmd.Run()
+}

pkg/core/backup/timer/timer.go

@@ -13,7 +13,7 @@ import (
 )
 
 // Start to backup with ticker
-func Start() error {
+func Start(restore bool) (err error) {
 	var storage Storage
 	// backup instance
 	switch config.Conf.BackupApp.BackupTo {
@@ -24,13 +24,17 @@ func Start() error {
 		return errors.New("timer: unknown backup to driver: " +
 			config.Conf.BackupApp.BackupTo)
 	}
-
+	if restore {
+		err = storage.RestoreData()
+		if err != nil {
+			return err
+		}
+	}
 	// parse duration
 	interval, err := ParseDuration(config.Conf.BackupApp.Interval)
 	if err != nil {
 		return err
 	}
-
 	t := time.NewTicker(interval)
 	for now := range t.C {
 		err = storage.BackupData(now)
@@ -65,4 +69,5 @@ func ParseDuration(d string) (time.Duration, error) {
 // Storage backup backend
 type Storage interface {
 	BackupData(now time.Time) error
+	RestoreData() error
 }

pkg/internal/qiniu.go

@@ -5,7 +5,9 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net/http"
 	"path/filepath"
+	"time"
 
 	"github.com/eiblog/eiblog/pkg/config"
 
@@ -15,9 +17,10 @@ import (
 
 // UploadParams upload params
 type UploadParams struct {
-	Name string
+	Name           string
-	Size int64
+	Size           int64
-	Data io.Reader
+	Data           io.Reader
+	NoCompletePath bool
 
 	Conf config.Qiniu
 }
@@ -28,7 +31,10 @@ func QiniuUpload(params UploadParams) (string, error) {
 		params.Conf.SecretKey == "" {
 		return "", errors.New("qiniu config error")
 	}
-	key := completeQiniuKey(params.Name)
+	key := params.Name
+	if !params.NoCompletePath {
+		key = filepath.Base(params.Name)
+	}
 
 	mac := qbox.NewMac(params.Conf.AccessKey,
 		params.Conf.SecretKey)
@@ -65,15 +71,19 @@ func QiniuUpload(params UploadParams) (string, error) {
 
 // DeleteParams delete params
 type DeleteParams struct {
-	Name string
+	Name           string
-	Days int
+	Days           int
+	NoCompletePath bool
 
 	Conf config.Qiniu
 }
 
 // QiniuDelete 删除文件
 func QiniuDelete(params DeleteParams) error {
-	key := completeQiniuKey(params.Name)
+	key := params.Name
+	if !params.NoCompletePath {
+		key = completeQiniuKey(params.Name)
+	}
 
 	mac := qbox.NewMac(params.Conf.AccessKey,
 		params.Conf.SecretKey)
@@ -95,6 +105,47 @@ func QiniuDelete(params DeleteParams) error {
 	return bucketManager.Delete(params.Conf.Bucket, key)
 }
 
+// ContentParams list params
+type ContentParams struct {
+	Prefix string
+
+	Conf config.Qiniu
+}
+
+// QiniuContent 获取文件列表
+func QiniuContent(params ContentParams) ([]byte, error) {
+	mac := qbox.NewMac(params.Conf.AccessKey,
+		params.Conf.SecretKey)
+	// region
+	region, err := storage.GetRegion(params.Conf.AccessKey, params.Conf.Bucket)
+	if err != nil {
+		return nil, err
+	}
+	cfg := &storage.Config{
+		UseHTTPS: true,
+		Region:   region,
+	}
+	// manager
+	bucketManager := storage.NewBucketManager(mac, cfg)
+	// list file
+	files, _, _, _, err := bucketManager.ListFiles(params.Conf.Bucket, params.Prefix, "", "", 2)
+	if err != nil {
+		return nil, err
+	}
+	if len(files) == 0 {
+		return nil, errors.New("no file")
+	}
+	deadline := time.Now().Add(time.Second * 60).Unix()
+	url := storage.MakePrivateURLv2(mac, "https://"+params.Conf.Domain, files[0].Key, deadline)
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return io.ReadAll(resp.Body)
+}
+
 // completeQiniuKey 修复路径
 func completeQiniuKey(name string) string {
 	ext := filepath.Ext(name)

pkg/internal/qiniu_test.go

@@ -46,3 +46,18 @@ func TestQiniuUpload(t *testing.T) {
 		})
 	}
 }
+
+func TestQiniuContent(t *testing.T) {
+	params := ContentParams{
+		Conf: config.Qiniu{
+			AccessKey: os.Getenv("QINIU_ACCESSKEY"),
+			SecretKey: os.Getenv("QINIU_SECRETKEY"),
+			Bucket:    os.Getenv("QINIU_BUCKET"),
+			Domain:    "bu.st.deepzz.com",
+		},
+	}
+	_, err := QiniuContent(params)
+	if err != nil {
+		t.Errorf("QiniuList error = %v", err)
+	}
+}