From 8e72c6228199d7beba941641ad986ce8c077ca29 Mon Sep 17 00:00:00 2001 From: Zheng Kai Date: Fri, 31 Mar 2023 10:12:18 +0800 Subject: [PATCH] up --- misc/docker/Dockerfile | 8 +++-- misc/docker/Makefile | 21 +++++++++---- misc/docker/install.sh | 15 ++++++---- misc/test/curl.sh | 6 ++++ server/src/config/config.go | 5 ++-- server/src/config/init.go | 4 ++- server/src/core/cache.go | 25 ++++++++++++++++ server/src/core/{rsp.go => fetch.go} | 8 +++-- server/src/core/metrics.go | 32 ++++++++++++++++++++ server/src/core/req.go | 28 ++++++++++++++++- server/src/core/row.go | 8 ++--- server/src/core/web.go | 37 ++++++++++++----------- server/src/metrics/init.go | 7 +++++ server/src/metrics/rsp.go | 45 ++++++++++++++++++++++++++++ server/src/util/ip.go | 22 +++++++------- server/src/web/server.go | 3 ++ server/src/zj/init.go | 16 +++++----- 17 files changed, 229 insertions(+), 61 deletions(-) create mode 100755 misc/test/curl.sh create mode 100644 server/src/core/cache.go rename server/src/core/{rsp.go => fetch.go} (85%) create mode 100644 server/src/core/metrics.go create mode 100644 server/src/metrics/rsp.go diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile index 5c77594..494f7c3 100644 --- a/misc/docker/Dockerfile +++ b/misc/docker/Dockerfile @@ -5,7 +5,7 @@ FROM golang:latest as builder ARG DOCKER_RUNNING=yes ARG DEBIAN_FRONTEND=noninteractive RUN apt update && apt install -yq protobuf-compiler tzdata ca-certificates -RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 COPY . /project @@ -23,6 +23,10 @@ COPY --from=builder /project/server/dist/prod/orca-server-next /orca-server RUN apk add --no-cache gzip brotli -ENV TZ=Asia/Shanghai +ENV TZ="Asia/Shanghai" + +ENV ORCA_WEB=":80" +ENV ORCA_LOG="/log" +ENV STATIC_DIR="/tmp" CMD ["/orca-server"] diff --git a/misc/docker/Makefile b/misc/docker/Makefile index 8b5a8ea..4b6d4b0 100644 --- a/misc/docker/Makefile +++ b/misc/docker/Makefile @@ -1,19 +1,28 @@ SHELL:=/bin/bash +include ../../server/build/env.sh + build: git sudo docker build -t orca -f Dockerfile ../.. run: build - sudo docker run --env "ORCA_MYSQL=orca:orca@tcp(172.17.0.1:3306)/orca" \ - --mount type=bind,source=/tmp/orca/tmp,target=/tmp \ - --mount type=bind,source=/tmp/orca/server/dist/prod/log,target=/log \ + sudo docker run \ + --env "OPENAI_API_KEY=$(OPENAI_API_KEY)" \ + --mount type=bind,source=/www/orca/static,target=/tmp \ + --mount type=bind,source=/www/orca/server/dist/prod/log,target=/log \ + -p 127.0.0.1:21035:80 \ orca install: build sudo docker save orca > docker-orca.tar - scp docker-orca.tar freya:/tmp - scp install.sh freya:/tmp - ssh freya "chmod +x /tmp/install.sh && /tmp/install.sh && rm /tmp/install.sh" + scp docker-orca.tar lamia:/tmp + scp install.sh lamia:/tmp + scp ../../server/build/env.sh lamia:/tmp + ssh lamia "chmod +x /tmp/install.sh && /tmp/install.sh && rm /tmp/install.sh" + ssh lamia "rm /tmp/env.sh" + +nginx: + scp ../nginx/prod.conf lamia:/etc/nginx/vhost.d/600-orca git: ../../server/build/git-hash.sh > ../../server/build/.git-hash diff --git a/misc/docker/install.sh b/misc/docker/install.sh index d568248..3aec660 100644 --- a/misc/docker/install.sh +++ b/misc/docker/install.sh @@ -1,12 +1,19 @@ #!/bin/bash -TARGET="Freya" +TARGET="Lamia" if [ "$HOSTNAME" != "$TARGET" ]; then >&2 echo only run in server "$TARGET" exit 1 fi +cd "$(dirname "$(readlink -f "$0")")" || exit 1 +if [ ! -e ./env.sh ]; then + >&2 echo no env file + exit 1 +fi +. ./env.sh || exit 1 + sudo docker stop orca sudo docker rm orca sudo docker rmi orca @@ -14,11 +21,9 @@ sudo docker rmi orca sudo cat /tmp/docker-orca.tar | sudo docker load sudo docker run -d --name orca \ - --env "TANK_MYSQL=orca:orca@tcp(172.17.0.1:3306)/orca" \ - --env "STATIC_DIR=/tmp" \ - --env "OUTPUT_PATH=/output" \ - --mount type=bind,source=/www/orca/output,target=/output \ + --env "OPENAI_API_KEY=${OPENAI_API_KEY}" \ --mount type=bind,source=/www/orca/log,target=/log \ --mount type=bind,source=/www/orca/static,target=/tmp \ + -p 127.0.0.1:21035:80 \ --restart always \ orca diff --git a/misc/test/curl.sh b/misc/test/curl.sh new file mode 100755 index 0000000..22cfa81 --- /dev/null +++ b/misc/test/curl.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +curl http://localhost:21035/v1/engines/text-embedding-ada-002/embeddings \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -d '{"input": ["\u80fd\u91cf\u793c\u7269\u662f\u600e\u4e48\u56de\u4e8b\uff1f\u7528\u4e2d\u6587"], "encoding_format": "base64"}' diff --git a/server/src/config/config.go b/server/src/config/config.go index a86d48e..d38da25 100644 --- a/server/src/config/config.go +++ b/server/src/config/config.go @@ -2,8 +2,9 @@ package config // config var ( - Prod bool - Dir string + Prod bool + Dir string + LogDir string StaticDir = `/www/orca/static` diff --git a/server/src/config/init.go b/server/src/config/init.go index 031ea24..f70df87 100644 --- a/server/src/config/init.go +++ b/server/src/config/init.go @@ -8,11 +8,13 @@ import ( func init() { Dir, _ = filepath.Abs(filepath.Dir(os.Args[0])) + LogDir = Dir + `/log` list := map[string]*string{ `OPENAI_API_KEY`: &OpenAIKey, `STATIC_DIR`: &StaticDir, - `WEB_ADDR`: &WebAddr, + `ORCA_WEB`: &WebAddr, + `ORCA_LOG`: &LogDir, } for k, v := range list { s := os.Getenv(k) diff --git a/server/src/core/cache.go b/server/src/core/cache.go new file mode 100644 index 0000000..fac33a8 --- /dev/null +++ b/server/src/core/cache.go @@ -0,0 +1,25 @@ +package core + +import ( + "project/pb" + "project/util" +) + +func tryCache(p *pb.Req) ([]byte, bool) { + + file := rspCacheFile(p) + if !util.FileExists(file) { + return nil, false + } + + ab, err := util.ReadFile(file) + if err != nil { + return nil, false + } + + return ab, true +} + +func rspCacheFile(r *pb.Req) string { + return util.CacheName(r.Hash()) + `-rsp.json` +} diff --git a/server/src/core/rsp.go b/server/src/core/fetch.go similarity index 85% rename from server/src/core/rsp.go rename to server/src/core/fetch.go index 29ff5d5..f3bd03c 100644 --- a/server/src/core/rsp.go +++ b/server/src/core/fetch.go @@ -36,7 +36,11 @@ func fetchRemote(r *pb.Req) (ab []byte, err error) { return } - defer rsp.Body.Close() + for k, v := range rsp.Header { + zj.J(k, v) + } - return io.ReadAll(rsp.Body) + ab, err = io.ReadAll(rsp.Body) + rsp.Body.Close() + return } diff --git a/server/src/core/metrics.go b/server/src/core/metrics.go new file mode 100644 index 0000000..89dd8e2 --- /dev/null +++ b/server/src/core/metrics.go @@ -0,0 +1,32 @@ +package core + +import ( + "encoding/json" + "net/http" + "project/metrics" + "project/pb" + "project/util" +) + +func doMetrics(ab []byte, cached bool, r *http.Request) { + + metrics.RspBytes(len(ab)) + + o := &pb.Rsp{} + json.Unmarshal(ab, o) + + u := o.GetUsage() + if u == nil { + metrics.RspJSONFail() + return + } + + metrics.RspToken(u.PromptTokens, u.TotalTokens, cached) + + ip, err := util.GetIP(r) + sip := ip.String() + if err != nil { + sip = `unknown` + } + metrics.RspTokenByIP(sip, u.TotalTokens) +} diff --git a/server/src/core/req.go b/server/src/core/req.go index ffc383f..b1b0281 100644 --- a/server/src/core/req.go +++ b/server/src/core/req.go @@ -8,12 +8,38 @@ import ( "project/zj" ) +func (c *Core) getAB(p *pb.Req, r *http.Request) (ab []byte, cached bool, err error) { + ab, ok := tryCache(p) + if ok { + cached = true + return + } + + pr := c.add(p, r) + + go func() { + reqFile := util.CacheName(p.Hash()) + `-req.json` + if !util.FileExists(reqFile) { + util.WriteFile(reqFile, p.Body) + } + }() + + pr.wait() + + ab = pr.rsp + err = pr.err + return +} + func req(w http.ResponseWriter, r *http.Request) (p *pb.Req, err error) { path := r.URL.Path method := r.Method - zj.J(method, path) + if path == `/favicon.ico` { + err = errSkip + return + } ab, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1024*1024)) if err != nil { diff --git a/server/src/core/row.go b/server/src/core/row.go index f5d911a..2283cc2 100644 --- a/server/src/core/row.go +++ b/server/src/core/row.go @@ -29,6 +29,7 @@ func (pr *row) run() { zj.J(`new`, s) pr.rsp, pr.err = fetchRemote(pr.req) + go pr.saveFile() go pr.metrics() @@ -44,11 +45,8 @@ func (pr *row) wait() { } func (pr *row) saveFile() { - rspFile := util.CacheName(pr.req.Hash()) + `-rsp.json` - if !util.FileExists(rspFile) { - util.WriteFile(rspFile, pr.rsp) - zj.J(rspFile) - } + rspFile := rspCacheFile(pr.req) + util.WriteFile(rspFile, pr.rsp) } func (pr *row) metrics() { diff --git a/server/src/core/web.go b/server/src/core/web.go index 2a754b7..1d708d7 100644 --- a/server/src/core/web.go +++ b/server/src/core/web.go @@ -1,36 +1,37 @@ package core import ( + "errors" "net/http" "project/metrics" - "project/util" + "project/zj" ) +var errSkip = errors.New(`skip`) + // WebHandle ... func (c *Core) WebHandle(w http.ResponseWriter, r *http.Request) { p, err := req(w, r) if err != nil { - metrics.ReqFailCount() - return - } - metrics.ReqBytes(len(p.Body)) - - pr := c.add(p, r) - - go func() { - reqFile := util.CacheName(p.Hash()) + `-req.json` - if !util.FileExists(reqFile) { - util.WriteFile(reqFile, p.Body) + if err != errSkip { + metrics.ReqFailCount() } - }() - - pr.wait() - - if pr.err != nil { err500(w) return } - w.Write(pr.rsp) + metrics.ReqBytes(len(p.Body)) + + ab, cached, err := c.getAB(p, r) + if err != nil { + err500(w) + return + } + zj.J(`cached`, cached) + + w.Header().Add(`Content-Type`, `application/json`) + w.Write(ab) + + go doMetrics(ab, cached, r) } diff --git a/server/src/metrics/init.go b/server/src/metrics/init.go index 151a7e7..a3e9f1e 100644 --- a/server/src/metrics/init.go +++ b/server/src/metrics/init.go @@ -7,4 +7,11 @@ func init() { prometheus.MustRegister(reqFailCount) prometheus.MustRegister(reqBytes) prometheus.MustRegister(errorCount) + + prometheus.MustRegister(rspBytes) + prometheus.MustRegister(rspPromptTokenCount) + prometheus.MustRegister(rspTokenCount) + prometheus.MustRegister(rspTokenCachedCount) + prometheus.MustRegister(rspJSONFailCount) + prometheus.MustRegister(rspTokenByIP) } diff --git a/server/src/metrics/rsp.go b/server/src/metrics/rsp.go new file mode 100644 index 0000000..089332d --- /dev/null +++ b/server/src/metrics/rsp.go @@ -0,0 +1,45 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + rspBytes = newCounter(`orca_rsp_bytes`, `rsp bytes`) + rspPromptTokenCount = newCounter(`orca_rsp_prompt_token`, `prompt token`) + rspTokenCount = newCounter(`orca_rsp_token`, `token`) + rspTokenCachedCount = newCounter(`orca_rsp_token_cached`, `token cached`) + rspJSONFailCount = newCounter(`orca_rsp_json_fail`, `json fail`) + rspTokenByIP = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: `orca_token_by_ip`, + Help: `API 返回报错`, + }, + []string{`ip`}, + ) +) + +// RspToken ... +func RspToken(prompt, total uint32, cached bool) { + if cached { + rspTokenCachedCount.Add(float64(total)) + return + } + rspPromptTokenCount.Add(float64(prompt)) + rspTokenCount.Add(float64(total)) +} + +// RspBytes ... +func RspBytes(n int) { + rspBytes.Add(float64(n)) +} + +// RspJSONFail ... +func RspJSONFail() { + rspJSONFailCount.Inc() +} + +// RspTokenByIP ... +func RspTokenByIP(ip string, token uint32) { + rspTokenByIP.WithLabelValues(ip).Add(float64(token)) +} diff --git a/server/src/util/ip.go b/server/src/util/ip.go index 7f9b1e8..34c7178 100644 --- a/server/src/util/ip.go +++ b/server/src/util/ip.go @@ -28,18 +28,20 @@ func GetIP(r *http.Request) (net.IP, error) { return nil, errors.New(`Invalid IP address`) } - // 检查是否是IPv4 - parsedIPv4 := parsedIP.To4() - if parsedIPv4 == nil { - return nil, errors.New(`IP address not IPv4`) - } + /* + // 检查是否是IPv4 + parsedIPv4 := parsedIP.To4() + if parsedIPv4 == nil { + return nil, errors.New(`IP address not IPv4`) + } - // 检查是否为局域网IP - if !parsedIP.IsPrivate() { - return nil, errors.New(`Public IP address not allowed`) - } + // 检查是否为局域网IP + if !parsedIP.IsPrivate() { + return nil, errors.New(`Public IP address not allowed`) + } + */ - return parsedIPv4, nil + return parsedIP, nil } // IPString ... diff --git a/server/src/web/server.go b/server/src/web/server.go index c1f8b7d..e4ef2b5 100644 --- a/server/src/web/server.go +++ b/server/src/web/server.go @@ -6,6 +6,8 @@ import ( "project/core" "project/zj" "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" ) // Server ... @@ -13,6 +15,7 @@ func Server() { mux := http.NewServeMux() + mux.Handle(`/_metrics`, promhttp.Handler()) mux.HandleFunc(`/`, core.NewCore().WebHandle) s := &http.Server{ diff --git a/server/src/zj/init.go b/server/src/zj/init.go index 994297e..5fdb61d 100644 --- a/server/src/zj/init.go +++ b/server/src/zj/init.go @@ -34,15 +34,13 @@ func Init() { baseLog.SetDirPrefix(filepath.Dir(zog.GetSourceFileDir())) - // 生产环境走 docker，不写本地文件 - if !config.Prod { + dir := config.LogDir - mainFile, _ := zog.NewFile(config.Dir+`/log/default.txt`, false) - infoFile, _ := zog.NewFile(config.Dir+`/log/io.txt`, false) - errFile, _ := zog.NewFile(config.Dir+`/log/err.txt`, true) + mainFile, _ := zog.NewFile(dir+`/default.txt`, false) + infoFile, _ := zog.NewFile(dir+`/io.txt`, false) + errFile, _ := zog.NewFile(dir+`/err.txt`, true) - mainCfg.Output = append(mainCfg.Output, mainFile) - infoCfg.Output = append(infoCfg.Output, infoFile) - errCfg.Output = append(errCfg.Output, mainFile, errFile) - } + mainCfg.Output = append(mainCfg.Output, mainFile) + infoCfg.Output = append(infoCfg.Output, infoFile) + errCfg.Output = append(errCfg.Output, mainFile, errFile) }