up

misc/test/chat-lamia.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -ex
+
+API_HOST="http://10.0.84.49:22035"
+
+curl "${API_HOST}/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $OPENAI_API_KEY" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "Hello!"}]
+  }'

misc/test/chat.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -ex
+
+API_HOST="http://localhost:22035"
+
+curl "${API_HOST}/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $OPENAI_API_KEY" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "Hello!"}]
+  }'

misc/test/curl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -ex
 
-API_HOST="http://10.0.84.49:22035"
-# API_HOST="http://localhost:22035"
+# API_HOST="http://10.0.84.49:22035"
+API_HOST="http://localhost:22035"
 
 curl "${API_HOST}/v1/engines/text-embedding-ada-002/embeddings" \
   -H "Content-Type: application/json" \

proto/rsp.proto

@@ -3,7 +3,7 @@ option go_package = "/pb";
 package pb;
 
 message Rsp {
-	uint32 id = 1;
+	string id = 1;
 	string object = 2;
 	fixed32 created = 3;
 	string model = 4;

server/src/core/metrics.go

@@ -7,6 +7,7 @@ import (
 	"project/pb"
 	"project/util"
 	"project/zj"
+	"strings"
 )
 
 func doMetrics(ab []byte, cached bool, r *http.Request) {
@@ -17,6 +18,7 @@ func doMetrics(ab []byte, cached bool, r *http.Request) {
 	err := json.Unmarshal(ab, o)
 	if err != nil {
 		zj.J(`unmarshal fail`, err)
+		util.WriteFile(`metrics-json-fail`, ab)
 		return
 	}
 
@@ -31,6 +33,10 @@ func doMetrics(ab []byte, cached bool, r *http.Request) {
 		zj.J(`token`, u.PromptTokens, u.TotalTokens)
 	}
 
+	metrics.RspTokenByModel(o.Model, u.TotalTokens)
+
+	metrics.RspTokenByKey(strings.TrimPrefix(r.Header.Get(`Authorization`), `Bearer `), u.TotalTokens)
+
 	ip, err := util.GetIP(r)
 	sip := ip.String()
 	if err != nil {

server/src/core/req.go

@@ -12,7 +12,7 @@ func (c *Core) getAB(p *pb.Req, r *http.Request) (ab []byte, cached bool, err er
 
 	canCache := p.Method != http.MethodGet && p.Method != http.MethodDelete
 
-	canCache = false
+	// canCache = false
 
 	if canCache {
 		var ok bool

server/src/metrics/init.go

@@ -16,4 +16,6 @@ func init() {
 	prometheus.MustRegister(rspTokenCachedCount)
 	prometheus.MustRegister(rspJSONFailCount)
 	prometheus.MustRegister(rspTokenByIP)
+	prometheus.MustRegister(rspTokenByModel)
+	prometheus.MustRegister(rspTokenByKey)
 }

server/src/metrics/rsp.go

@@ -10,10 +10,24 @@ var (
 	rspTokenCount       = newCounter(`orca_rsp_token`, `token`)
 	rspTokenCachedCount = newCounter(`orca_rsp_token_cached`, `token cached`)
 	rspJSONFailCount    = newCounter(`orca_rsp_json_fail`, `json fail`)
+	rspTokenByModel     = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: `orca_token_by_model`,
+			Help: `token by model`,
+		},
+		[]string{`model`},
+	)
+	rspTokenByKey = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: `orca_token_by_key`,
+			Help: `openai key`,
+		},
+		[]string{`key`},
+	)
 	rspTokenByIP = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: `orca_token_by_ip`,
-			Help: `API 返回报错`,
+			Help: `token by ip`,
 		},
 		[]string{`ip`},
 	)
@@ -43,3 +57,18 @@ func RspJSONFail() {
 func RspTokenByIP(ip string, token uint32) {
 	rspTokenByIP.WithLabelValues(ip).Add(float64(token))
 }
+
+// RspTokenByKey ...
+func RspTokenByKey(key string, token uint32) {
+	if len(key) > 30 {
+		key = key[:30]
+	} else if key == `` {
+		key = `<empty>`
+	}
+	rspTokenByKey.WithLabelValues(key).Add(float64(token))
+}
+
+// RspTokenByModel ...
+func RspTokenByModel(model string, token uint32) {
+	rspTokenByModel.WithLabelValues(model).Add(float64(token))
+}