From 51e0b5807a7d25bc87a1cd5e067b851ace88d965 Mon Sep 17 00:00:00 2001 From: Zheng Kai Date: Fri, 9 Jun 2023 11:09:14 +0800 Subject: [PATCH] up --- misc/test/chat-lamia.sh | 11 +++++++++++ misc/test/chat.sh | 11 +++++++++++ misc/test/curl.sh | 4 ++-- proto/rsp.proto | 2 +- server/src/core/metrics.go | 6 ++++++ server/src/core/req.go | 2 +- server/src/metrics/init.go | 2 ++ server/src/metrics/rsp.go | 33 +++++++++++++++++++++++++++++++-- 8 files changed, 65 insertions(+), 6 deletions(-) create mode 100755 misc/test/chat-lamia.sh create mode 100755 misc/test/chat.sh diff --git a/misc/test/chat-lamia.sh b/misc/test/chat-lamia.sh new file mode 100755 index 0000000..d21c33e --- /dev/null +++ b/misc/test/chat-lamia.sh @@ -0,0 +1,11 @@ +#!/bin/bash -ex + +API_HOST="http://10.0.84.49:22035" + +curl "${API_HOST}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello!"}] + }' diff --git a/misc/test/chat.sh b/misc/test/chat.sh new file mode 100755 index 0000000..32f055f --- /dev/null +++ b/misc/test/chat.sh @@ -0,0 +1,11 @@ +#!/bin/bash -ex + +API_HOST="http://localhost:22035" + +curl "${API_HOST}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello!"}] + }' diff --git a/misc/test/curl.sh b/misc/test/curl.sh index a55c2fd..f0dd9dd 100755 --- a/misc/test/curl.sh +++ b/misc/test/curl.sh @@ -1,7 +1,7 @@ #!/bin/bash -ex -API_HOST="http://10.0.84.49:22035" -# API_HOST="http://localhost:22035" +# API_HOST="http://10.0.84.49:22035" +API_HOST="http://localhost:22035" curl "${API_HOST}/v1/engines/text-embedding-ada-002/embeddings" \ -H "Content-Type: application/json" \ diff --git a/proto/rsp.proto b/proto/rsp.proto index 867bed4..953dc42 100644 --- a/proto/rsp.proto +++ b/proto/rsp.proto @@ -3,7 +3,7 @@ option go_package = "/pb"; package pb; message Rsp { - uint32 id = 1; + string id = 1; string object = 2; fixed32 created = 3; string model = 4; diff --git a/server/src/core/metrics.go b/server/src/core/metrics.go index d739d5e..50f3330 100644 --- a/server/src/core/metrics.go +++ b/server/src/core/metrics.go @@ -7,6 +7,7 @@ import ( "project/pb" "project/util" "project/zj" + "strings" ) func doMetrics(ab []byte, cached bool, r *http.Request) { @@ -17,6 +18,7 @@ func doMetrics(ab []byte, cached bool, r *http.Request) { err := json.Unmarshal(ab, o) if err != nil { zj.J(`unmarshal fail`, err) + util.WriteFile(`metrics-json-fail`, ab) return } @@ -31,6 +33,10 @@ func doMetrics(ab []byte, cached bool, r *http.Request) { zj.J(`token`, u.PromptTokens, u.TotalTokens) } + metrics.RspTokenByModel(o.Model, u.TotalTokens) + + metrics.RspTokenByKey(strings.TrimPrefix(r.Header.Get(`Authorization`), `Bearer `), u.TotalTokens) + ip, err := util.GetIP(r) sip := ip.String() if err != nil { diff --git a/server/src/core/req.go b/server/src/core/req.go index 5784865..7b672b4 100644 --- a/server/src/core/req.go +++ b/server/src/core/req.go @@ -12,7 +12,7 @@ func (c *Core) getAB(p *pb.Req, r *http.Request) (ab []byte, cached bool, err er canCache := p.Method != http.MethodGet && p.Method != http.MethodDelete - canCache = false + // canCache = false if canCache { var ok bool diff --git a/server/src/metrics/init.go b/server/src/metrics/init.go index 9e68b39..81c62b1 100644 --- a/server/src/metrics/init.go +++ b/server/src/metrics/init.go @@ -16,4 +16,6 @@ func init() { prometheus.MustRegister(rspTokenCachedCount) prometheus.MustRegister(rspJSONFailCount) prometheus.MustRegister(rspTokenByIP) + prometheus.MustRegister(rspTokenByModel) + prometheus.MustRegister(rspTokenByKey) } diff --git a/server/src/metrics/rsp.go b/server/src/metrics/rsp.go index 089332d..f6aecf9 100644 --- a/server/src/metrics/rsp.go +++ b/server/src/metrics/rsp.go @@ -10,10 +10,24 @@ var ( rspTokenCount = newCounter(`orca_rsp_token`, `token`) rspTokenCachedCount = newCounter(`orca_rsp_token_cached`, `token cached`) rspJSONFailCount = newCounter(`orca_rsp_json_fail`, `json fail`) - rspTokenByIP = prometheus.NewCounterVec( + rspTokenByModel = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: `orca_token_by_model`, + Help: `token by model`, + }, + []string{`model`}, + ) + rspTokenByKey = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: `orca_token_by_key`, + Help: `openai key`, + }, + []string{`key`}, + ) + rspTokenByIP = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: `orca_token_by_ip`, - Help: `API θΏ”ε›žζŠ₯ι”™`, + Help: `token by ip`, }, []string{`ip`}, ) @@ -43,3 +57,18 @@ func RspJSONFail() { func RspTokenByIP(ip string, token uint32) { rspTokenByIP.WithLabelValues(ip).Add(float64(token)) } + +// RspTokenByKey ... +func RspTokenByKey(key string, token uint32) { + if len(key) > 30 { + key = key[:30] + } else if key == `` { + key = `` + } + rspTokenByKey.WithLabelValues(key).Add(float64(token)) +} + +// RspTokenByModel ... +func RspTokenByModel(model string, token uint32) { + rspTokenByModel.WithLabelValues(model).Add(float64(token)) +}