up

misc/docker/Dockerfile

@@ -9,8 +9,6 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0
 
 COPY . /project
 
-RUN cd /project/proto && make
-
 RUN /project/server/build/build-server.sh prod
 
 # clean stage

proto/es.proto

@@ -20,3 +20,63 @@ message EsMetricsToken {
 	uint32 completion = 2;
 	uint32 prompt = 3;
 }
+
+message EsErrorTry {
+	EsError error = 1;
+	uint32 status = 2;
+}
+
+message EsError {
+	repeated EsErrorRootCause root_cause = 1;
+	string type = 2;
+	string reason = 3;
+	uint32 line = 4;
+	uint32 col = 5;
+}
+
+message EsErrorRootCause {
+	string type = 1;
+	string reason = 2;
+	uint32 line = 3;
+	uint32 col = 4;
+}
+
+message EsResultDateHistogram {
+	EsResultDateHistogramAggregations aggregations = 1;
+}
+
+message EsResultDateHistogramAggregations {
+	EsResultDateHistogramSum bytes_sum = 1;
+}
+
+message EsResultDateHistogramSum {
+	repeated EsResultDateHistogramBucket buckets = 1;
+}
+
+message EsResultDateHistogramBucket {
+	EsResultValueInt reqBytes = 1;
+	EsResultValueInt rspBytes = 2;
+	EsResultValueInt tokenTotal = 3;
+	EsResultValueInt tokenCompletion = 4;
+	EsResultValueInt tokenPrompt = 5;
+	uint32 doc_count = 6;
+	uint64 key = 7;
+}
+
+message EsDateHistogramList {
+	repeated EsDateHistogram list = 1;
+}
+
+message EsDateHistogram {
+	uint32 reqBytes = 1;
+	uint32 rspBytes = 2;
+	uint32 tokenTotal = 3;
+	uint32 tokenCompletion = 4;
+	uint32 tokenPrompt = 5;
+	uint32 count = 6;
+	uint32 ts = 7;
+}
+
+message EsResultValueInt {
+	uint32 value = 1;
+}

server/build/build-server.sh

@@ -9,6 +9,10 @@ fi
 
 . ./common.sh
 
+if [ ! -e ../src/pb/req.pb.go ]; then
+	(cd ../../proto && protoc --go_out=../server/src ./*.proto)
+fi
+
 DATE=$(TZ='Asia/Shanghai' date '+%Y-%m-%d %H:%M:%S')
 GO_VERSION=$(go version)
 GIT_VERSION=$(./git-hash.sh)

server/src/es/index.go

@@ -15,18 +15,22 @@ import (
 //go:embed tpl/mapping.json
 var indexMapping string
 
-func indexName(ts uint32) string {
-
+func indexNameBase() string {
 	index := `orca-metrics`
 	if !config.Prod {
 		index = `dev-` + index
 	}
-
-	index = fmt.Sprintf(`%s-%s`, index, time.Unix(int64(ts), 0).Format(`2006.01.02`))
-
 	return index
 }
 
+func indexName(ts uint32) string {
+	return fmt.Sprintf(`%s-%s`, indexNameBase(), time.Unix(int64(ts), 0).Format(`2006.01.02`))
+}
+
+func indexNameAll() string {
+	return indexNameBase() + `-*`
+}
+
 func createIndex() {
 
 	ts := zu.TS()

server/src/es/search.go

@@ -0,0 +1,75 @@
+package es
+
+import (
+	"context"
+	"errors"
+	"io"
+	"project/pb"
+	"project/util"
+	"project/zj"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+)
+
+// LastItem ...
+func LastItem() {
+
+	query := `{
+		"size": 100,
+		"sort": [
+			{
+				"ts": {
+					"order": "desc"
+				}
+			}
+		]
+	}`
+	zj.J(query)
+
+	/*
+		ab, err := Search(query)
+		if err != nil {
+			zj.W(`fail`, err)
+			return
+		}
+
+		util.WriteFile(`search.json`, ab)
+
+		zj.J(string(ab))
+	*/
+}
+
+// Search ...
+func Search(query string, m proto.Message) error {
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	re, err := theClient.Search(
+		theClient.Search.WithContext(ctx),
+		theClient.Search.WithIndex(indexNameAll()),
+		theClient.Search.WithBody(strings.NewReader(query)),
+		theClient.Search.WithTrackTotalHits(false),
+	)
+	defer cancel()
+	if err != nil {
+		return err
+	}
+	defer re.Body.Close()
+
+	ab, err := io.ReadAll(re.Body)
+	if err != nil {
+		return err
+	}
+
+	et := &pb.EsErrorTry{}
+	util.JSONUnmarshal(ab, et)
+	e := et.GetError()
+	if e != nil {
+		return errors.New(e.Reason)
+	}
+
+	util.WriteFile(`last-search.json`, ab)
+
+	return util.JSONUnmarshal(ab, m)
+}

server/src/project.go

@@ -20,6 +20,8 @@ func Start() {
 	es.Init()
 
 	if !config.Prod {
+		// es.LastItem()
+		// st.DateHistogram()
 		go es.Test()
 	}
 

server/src/st/date-histogram.go

@@ -0,0 +1,53 @@
+package st
+
+import (
+	"fmt"
+	"project/es"
+	"project/pb"
+	"project/zj"
+
+	_ "embed" //
+
+	"github.com/zhengkai/zu"
+)
+
+//go:embed tpl/date-histogram.json
+var queryDateHistogramTpl string
+
+// DateHistogram ...
+func DateHistogram() {
+
+	query := fmt.Sprintf(
+		queryDateHistogramTpl,
+		`now-24h`,
+		`now`,
+	)
+
+	d := &pb.EsResultDateHistogram{}
+	err := es.Search(query, d)
+	if err != nil {
+		zj.W(`fail`, err)
+		return
+	}
+
+	li := d.GetAggregations().GetBytesSum().GetBuckets()
+	if len(li) == 0 {
+		return
+	}
+
+	re := make([]*pb.EsDateHistogram, len(li))
+	for i, v := range li {
+		re[i] = &pb.EsDateHistogram{
+			Ts:              uint32(v.GetKey() / 1000),
+			ReqBytes:        v.GetReqBytes().GetValue(),
+			RspBytes:        v.GetRspBytes().GetValue(),
+			TokenTotal:      v.GetTokenTotal().GetValue(),
+			TokenCompletion: v.GetTokenCompletion().GetValue(),
+			TokenPrompt:     v.GetTokenPrompt().GetValue(),
+		}
+	}
+
+	j := zu.JSONPretty(re)
+	// util.WriteFile(`date-histogram.json`, []byte(j))
+	zj.J(j)
+}

server/src/st/tpl/date-histogram.json

@@ -0,0 +1,46 @@
+{
+	"size": 0,
+		"query": {
+			"range": {
+				"ts": {
+					"gte": "%s",
+					"lt": "%s"
+				}
+			}
+		},
+		"aggs": {
+			"bytes_sum": {
+				"date_histogram": {
+					"field": "ts",
+					"calendar_interval": "1m"
+				},
+				"aggs": {
+					"tokenTotal": {
+						"sum": {
+							"field": "token.total"
+						}
+					},
+					"tokenCompletion": {
+						"sum": {
+							"field": "token.completion"
+						}
+					},
+					"tokenPrompt": {
+						"sum": {
+							"field": "token.prompt"
+						}
+					},
+					"reqBytes": {
+						"sum": {
+							"field": "reqBytes"
+						}
+					},
+					"rspBytes": {
+						"sum": {
+							"field": "rspBytes"
+						}
+					}
+				}
+			}
+		}
+}

server/src/st/web.go

@@ -0,0 +1,7 @@
+package st
+
+import "fmt"
+
+func main() {
+	fmt.Println("vim-go")
+}