From 05fe88de8bb4d569eefbfe5cc56c02b85c206bf0 Mon Sep 17 00:00:00 2001 From: Zheng Kai Date: Fri, 16 Jun 2023 14:59:16 +0800 Subject: [PATCH] up --- misc/docker/Dockerfile | 2 - proto/es.proto | 60 +++++++++++++++++++++ server/build/build-server.sh | 4 ++ server/src/es/index.go | 14 +++-- server/src/es/search.go | 75 +++++++++++++++++++++++++++ server/src/project.go | 2 + server/src/st/date-histogram.go | 53 +++++++++++++++++++ server/src/st/tpl/date-histogram.json | 46 ++++++++++++++++ server/src/st/web.go | 7 +++ 9 files changed, 256 insertions(+), 7 deletions(-) create mode 100644 server/src/es/search.go create mode 100644 server/src/st/date-histogram.go create mode 100644 server/src/st/tpl/date-histogram.json create mode 100644 server/src/st/web.go diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile index 21f8ad0..ffb90c6 100644 --- a/misc/docker/Dockerfile +++ b/misc/docker/Dockerfile @@ -9,8 +9,6 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 COPY . /project -RUN cd /project/proto && make - RUN /project/server/build/build-server.sh prod # clean stage diff --git a/proto/es.proto b/proto/es.proto index e33b17e..81c4b58 100644 --- a/proto/es.proto +++ b/proto/es.proto @@ -20,3 +20,63 @@ message EsMetricsToken { uint32 completion = 2; uint32 prompt = 3; } + +message EsErrorTry { + EsError error = 1; + uint32 status = 2; +} + +message EsError { + repeated EsErrorRootCause root_cause = 1; + string type = 2; + string reason = 3; + uint32 line = 4; + uint32 col = 5; +} + +message EsErrorRootCause { + string type = 1; + string reason = 2; + uint32 line = 3; + uint32 col = 4; +} + +message EsResultDateHistogram { + EsResultDateHistogramAggregations aggregations = 1; +} + +message EsResultDateHistogramAggregations { + EsResultDateHistogramSum bytes_sum = 1; +} + +message EsResultDateHistogramSum { + repeated EsResultDateHistogramBucket buckets = 1; +} + +message EsResultDateHistogramBucket { + EsResultValueInt reqBytes = 1; + EsResultValueInt rspBytes = 2; + EsResultValueInt tokenTotal = 3; + EsResultValueInt tokenCompletion = 4; + EsResultValueInt tokenPrompt = 5; + uint32 doc_count = 6; + uint64 key = 7; +} + +message EsDateHistogramList { + repeated EsDateHistogram list = 1; +} + +message EsDateHistogram { + uint32 reqBytes = 1; + uint32 rspBytes = 2; + uint32 tokenTotal = 3; + uint32 tokenCompletion = 4; + uint32 tokenPrompt = 5; + uint32 count = 6; + uint32 ts = 7; +} + +message EsResultValueInt { + uint32 value = 1; +} diff --git a/server/build/build-server.sh b/server/build/build-server.sh index 857e419..97457c2 100755 --- a/server/build/build-server.sh +++ b/server/build/build-server.sh @@ -9,6 +9,10 @@ fi . ./common.sh +if [ ! -e ../src/pb/req.pb.go ]; then + (cd ../../proto && protoc --go_out=../server/src ./*.proto) +fi + DATE=$(TZ='Asia/Shanghai' date '+%Y-%m-%d %H:%M:%S') GO_VERSION=$(go version) GIT_VERSION=$(./git-hash.sh) diff --git a/server/src/es/index.go b/server/src/es/index.go index 1d0d2af..d7dc648 100644 --- a/server/src/es/index.go +++ b/server/src/es/index.go @@ -15,18 +15,22 @@ import ( //go:embed tpl/mapping.json var indexMapping string -func indexName(ts uint32) string { - +func indexNameBase() string { index := `orca-metrics` if !config.Prod { index = `dev-` + index } - - index = fmt.Sprintf(`%s-%s`, index, time.Unix(int64(ts), 0).Format(`2006.01.02`)) - return index } +func indexName(ts uint32) string { + return fmt.Sprintf(`%s-%s`, indexNameBase(), time.Unix(int64(ts), 0).Format(`2006.01.02`)) +} + +func indexNameAll() string { + return indexNameBase() + `-*` +} + func createIndex() { ts := zu.TS() diff --git a/server/src/es/search.go b/server/src/es/search.go new file mode 100644 index 0000000..21aa7f9 --- /dev/null +++ b/server/src/es/search.go @@ -0,0 +1,75 @@ +package es + +import ( + "context" + "errors" + "io" + "project/pb" + "project/util" + "project/zj" + "strings" + "time" + + "google.golang.org/protobuf/proto" +) + +// LastItem ... +func LastItem() { + + query := `{ + "size": 100, + "sort": [ + { + "ts": { + "order": "desc" + } + } + ] + }` + zj.J(query) + + /* + ab, err := Search(query) + if err != nil { + zj.W(`fail`, err) + return + } + + util.WriteFile(`search.json`, ab) + + zj.J(string(ab)) + */ +} + +// Search ... +func Search(query string, m proto.Message) error { + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + re, err := theClient.Search( + theClient.Search.WithContext(ctx), + theClient.Search.WithIndex(indexNameAll()), + theClient.Search.WithBody(strings.NewReader(query)), + theClient.Search.WithTrackTotalHits(false), + ) + defer cancel() + if err != nil { + return err + } + defer re.Body.Close() + + ab, err := io.ReadAll(re.Body) + if err != nil { + return err + } + + et := &pb.EsErrorTry{} + util.JSONUnmarshal(ab, et) + e := et.GetError() + if e != nil { + return errors.New(e.Reason) + } + + util.WriteFile(`last-search.json`, ab) + + return util.JSONUnmarshal(ab, m) +} diff --git a/server/src/project.go b/server/src/project.go index 8124755..791e857 100644 --- a/server/src/project.go +++ b/server/src/project.go @@ -20,6 +20,8 @@ func Start() { es.Init() if !config.Prod { + // es.LastItem() + // st.DateHistogram() go es.Test() } diff --git a/server/src/st/date-histogram.go b/server/src/st/date-histogram.go new file mode 100644 index 0000000..332d41d --- /dev/null +++ b/server/src/st/date-histogram.go @@ -0,0 +1,53 @@ +package st + +import ( + "fmt" + "project/es" + "project/pb" + "project/zj" + + _ "embed" // + + "github.com/zhengkai/zu" +) + +//go:embed tpl/date-histogram.json +var queryDateHistogramTpl string + +// DateHistogram ... +func DateHistogram() { + + query := fmt.Sprintf( + queryDateHistogramTpl, + `now-24h`, + `now`, + ) + + d := &pb.EsResultDateHistogram{} + err := es.Search(query, d) + if err != nil { + zj.W(`fail`, err) + return + } + + li := d.GetAggregations().GetBytesSum().GetBuckets() + if len(li) == 0 { + return + } + + re := make([]*pb.EsDateHistogram, len(li)) + for i, v := range li { + re[i] = &pb.EsDateHistogram{ + Ts: uint32(v.GetKey() / 1000), + ReqBytes: v.GetReqBytes().GetValue(), + RspBytes: v.GetRspBytes().GetValue(), + TokenTotal: v.GetTokenTotal().GetValue(), + TokenCompletion: v.GetTokenCompletion().GetValue(), + TokenPrompt: v.GetTokenPrompt().GetValue(), + } + } + + j := zu.JSONPretty(re) + // util.WriteFile(`date-histogram.json`, []byte(j)) + zj.J(j) +} diff --git a/server/src/st/tpl/date-histogram.json b/server/src/st/tpl/date-histogram.json new file mode 100644 index 0000000..462d20b --- /dev/null +++ b/server/src/st/tpl/date-histogram.json @@ -0,0 +1,46 @@ +{ + "size": 0, + "query": { + "range": { + "ts": { + "gte": "%s", + "lt": "%s" + } + } + }, + "aggs": { + "bytes_sum": { + "date_histogram": { + "field": "ts", + "calendar_interval": "1m" + }, + "aggs": { + "tokenTotal": { + "sum": { + "field": "token.total" + } + }, + "tokenCompletion": { + "sum": { + "field": "token.completion" + } + }, + "tokenPrompt": { + "sum": { + "field": "token.prompt" + } + }, + "reqBytes": { + "sum": { + "field": "reqBytes" + } + }, + "rspBytes": { + "sum": { + "field": "rspBytes" + } + } + } + } + } +} diff --git a/server/src/st/web.go b/server/src/st/web.go new file mode 100644 index 0000000..84c40a8 --- /dev/null +++ b/server/src/st/web.go @@ -0,0 +1,7 @@ +package st + +import "fmt" + +func main() { + fmt.Println("vim-go") +}