up

misc/docker/Makefile

@@ -8,6 +8,9 @@ build: git
 run: build
 	sudo docker run \
 		--env "OPENAI_API_KEY=$(OPENAI_API_KEY)" \
+		--env "ORCA_ES_ADDR=$(ORCA_ES_ADDR)" \
+		--env "ORCA_ES_USER=$(ORCA_ES_USER)" \
+		--env "ORCA_ES_PASS=$(ORCA_ES_PASS)" \
 		--mount type=bind,source=/www/orca/static,target=/tmp \
 		--mount type=bind,source=/www/orca/server/dist/prod/log,target=/log \
 		-p 127.0.0.1:21035:80 \

proto/es.proto

@@ -12,6 +12,7 @@ message EsMetrics {
 	uint32 reqBytes = 7;
 	uint32 rspBytes = 8;
 	uint64 ts = 9;
+	string hash = 10;
 }
 
 message EsMetricsToken {

server/src/config/config.go

@@ -14,6 +14,7 @@ var (
 
 	OpenAIKey = ``
 
+	ESAddr = ``
 	ESUser = ``
 	ESPass = ``
 )

server/src/config/init.go

@@ -17,6 +17,7 @@ func init() {
 		`STATIC_DIR`:     &StaticDir,
 		`ORCA_WEB`:       &WebAddr,
 		`ORCA_LOG`:       &LogDir,
+		`ORCA_ES_ADDR`:   &ESAddr,
 		`ORCA_ES_USER`:   &ESUser,
 		`ORCA_ES_PASS`:   &ESPass,
 	}

server/src/es/init.go

@@ -12,10 +12,15 @@ var theClient *elasticsearch.Client
 // Init ...
 func Init() (err error) {
 
-	theClient, err = elasticsearch.NewClient(elasticsearch.Config{
+	cfg := elasticsearch.Config{
 		Username: config.ESUser,
 		Password: config.ESPass,
-	})
+	}
+	if config.ESAddr != `` {
+		cfg.Addresses = []string{config.ESAddr}
+	}
+
+	theClient, err = elasticsearch.NewClient(cfg)
 	if err != nil {
 		return err
 	}

server/src/es/insert.go

@@ -2,8 +2,9 @@ package es
 
 import (
 	"bytes"
-	"encoding/json"
 	"project/pb"
+
+	"google.golang.org/protobuf/encoding/protojson"
 )
 
 // Insert ...
@@ -13,7 +14,10 @@ func Insert(d *pb.EsMetrics) {
 		return
 	}
 
-	ab, err := json.Marshal(d)
+	ab, err := protojson.MarshalOptions{
+		EmitUnpopulated: true,
+		AllowPartial:    true,
+	}.Marshal(d)
 	if err != nil {
 		return
 	}

server/src/es/test.go

@@ -1,6 +1,7 @@
 package es
 
 import (
+	"crypto/md5"
 	"fmt"
 	"math/rand"
 	"project/pb"
@@ -14,6 +15,8 @@ func Test() {
 
 	for {
 
+		ts := zu.MS()
+
 		minute := time.Now().Minute() - 30
 		if minute < 0 {
 			minute = -minute
@@ -27,20 +30,37 @@ func Test() {
 		}
 		model := modelChoose[rand.Intn(len(modelChoose))]
 
+		ipChoose := []string{
+			`10.0.32.43`,
+			`10.0.84.231`,
+			`10.0.84.49`,
+			`10.2.197.197`,
+		}
+		ip := ipChoose[rand.Intn(len(ipChoose))]
+
+		keyChoose := []string{
+			`zhengkai.orca`,
+			`zhengkai.debug`,
+		}
+		key := keyChoose[rand.Intn(len(keyChoose))]
+
+		hash := md5.Sum([]byte(fmt.Sprintf(`%s-%s-%s-%d`, model, ip, key, ts)))
+
 		d := &pb.EsMetrics{
-			ID: fmt.Sprintf(`chatcmpl-%d`, zu.MS()),
+			ID: fmt.Sprintf(`chatcmpl-%d`, ts),
 			Token: &pb.EsMetricsToken{
 				Total:      uint32(rand.Intn(minute * 10)),
 				Completion: uint32(rand.Intn(minute * 10)),
 				Prompt:     uint32(rand.Intn(minute * 10)),
 			},
-			Cached:   true,
-			Ip:       `127.0.0.1`,
+			Cached:   rand.Intn(2) == 1,
+			Ip:       ip,
 			Model:    model,
-			Key:      `zhengkai.orca`,
+			Key:      key,
 			ReqBytes: uint32(rand.Intn(minute * 57)),
 			RspBytes: uint32(rand.Intn(minute * 21)),
-			Ts:       zu.MS(),
+			Ts:       ts,
+			Hash:     fmt.Sprintf(`%x`, hash),
 		}
 		Insert(d)
 

server/src/es/tpl/mapping.json

@@ -37,6 +37,9 @@
 			},
 			"ts": {
 				"type": "date"
+			},
+			"hash": {
+				"type": "keyword"
 			}
 		}
 	}