up

misc/docker/Dockerfile

@@ -5,7 +5,7 @@ FROM golang:latest as builder
 ARG DOCKER_RUNNING=yes
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt update && apt install -yq protobuf-compiler tzdata ca-certificates
-RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0
 
 COPY . /project
 
@@ -23,6 +23,10 @@ COPY --from=builder /project/server/dist/prod/orca-server-next /orca-server
 
 RUN apk add --no-cache gzip brotli
 
-ENV TZ=Asia/Shanghai
+ENV TZ="Asia/Shanghai"
+
+ENV ORCA_WEB=":80"
+ENV ORCA_LOG="/log"
+ENV STATIC_DIR="/tmp"
 
 CMD ["/orca-server"]

misc/docker/Makefile

@@ -1,19 +1,28 @@
 SHELL:=/bin/bash
 
+include ../../server/build/env.sh
+
 build: git
 	sudo docker build -t orca -f Dockerfile ../..
 
 run: build
-	sudo docker run --env "ORCA_MYSQL=orca:orca@tcp(172.17.0.1:3306)/orca" \
+	sudo docker run \
-		--mount type=bind,source=/tmp/orca/tmp,target=/tmp \
+		--env "OPENAI_API_KEY=$(OPENAI_API_KEY)" \
-		--mount type=bind,source=/tmp/orca/server/dist/prod/log,target=/log \
+		--mount type=bind,source=/www/orca/static,target=/tmp \
+		--mount type=bind,source=/www/orca/server/dist/prod/log,target=/log \
+		-p 127.0.0.1:21035:80 \
 		orca
 
 install: build
 	sudo docker save orca > docker-orca.tar
-	scp docker-orca.tar freya:/tmp
+	scp docker-orca.tar lamia:/tmp
-	scp install.sh freya:/tmp
+	scp install.sh lamia:/tmp
-	ssh freya "chmod +x /tmp/install.sh && /tmp/install.sh && rm /tmp/install.sh"
+	scp ../../server/build/env.sh lamia:/tmp
+	ssh lamia "chmod +x /tmp/install.sh && /tmp/install.sh && rm /tmp/install.sh"
+	ssh lamia "rm /tmp/env.sh"
+
+nginx:
+	scp ../nginx/prod.conf lamia:/etc/nginx/vhost.d/600-orca
 
 git:
 	../../server/build/git-hash.sh > ../../server/build/.git-hash

misc/docker/install.sh

@@ -1,12 +1,19 @@
 #!/bin/bash
 
-TARGET="Freya"
+TARGET="Lamia"
 
 if [ "$HOSTNAME" != "$TARGET" ]; then
 	>&2 echo only run in server "$TARGET"
 	exit 1
 fi
 
+cd "$(dirname "$(readlink -f "$0")")" || exit 1
+if [ ! -e ./env.sh ]; then
+	>&2 echo no env file
+	exit 1
+fi
+. ./env.sh || exit 1
+
 sudo docker stop orca
 sudo docker rm orca
 sudo docker rmi orca
@@ -14,11 +21,9 @@ sudo docker rmi orca
 sudo cat /tmp/docker-orca.tar | sudo docker load
 
 sudo docker run -d --name orca \
-	--env "TANK_MYSQL=orca:orca@tcp(172.17.0.1:3306)/orca" \
+	--env "OPENAI_API_KEY=${OPENAI_API_KEY}" \
-	--env "STATIC_DIR=/tmp" \
-	--env "OUTPUT_PATH=/output" \
-	--mount type=bind,source=/www/orca/output,target=/output \
 	--mount type=bind,source=/www/orca/log,target=/log \
 	--mount type=bind,source=/www/orca/static,target=/tmp \
+	-p 127.0.0.1:21035:80 \
 	--restart always \
 	orca

misc/test/curl.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+curl http://localhost:21035/v1/engines/text-embedding-ada-002/embeddings \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $OPENAI_API_KEY" \
+  -d '{"input": ["\u80fd\u91cf\u793c\u7269\u662f\u600e\u4e48\u56de\u4e8b\uff1f\u7528\u4e2d\u6587"], "encoding_format": "base64"}'

server/src/config/config.go

@@ -2,8 +2,9 @@ package config
 
 // config
 var (
-	Prod bool
+	Prod   bool
-	Dir  string
+	Dir    string
+	LogDir string
 
 	StaticDir = `/www/orca/static`
 

server/src/config/init.go

@@ -8,11 +8,13 @@ import (
 func init() {
 
 	Dir, _ = filepath.Abs(filepath.Dir(os.Args[0]))
+	LogDir = Dir + `/log`
 
 	list := map[string]*string{
 		`OPENAI_API_KEY`: &OpenAIKey,
 		`STATIC_DIR`:     &StaticDir,
-		`WEB_ADDR`:       &WebAddr,
+		`ORCA_WEB`:       &WebAddr,
+		`ORCA_LOG`:       &LogDir,
 	}
 	for k, v := range list {
 		s := os.Getenv(k)

server/src/core/cache.go

@@ -0,0 +1,25 @@
+package core
+
+import (
+	"project/pb"
+	"project/util"
+)
+
+func tryCache(p *pb.Req) ([]byte, bool) {
+
+	file := rspCacheFile(p)
+	if !util.FileExists(file) {
+		return nil, false
+	}
+
+	ab, err := util.ReadFile(file)
+	if err != nil {
+		return nil, false
+	}
+
+	return ab, true
+}
+
+func rspCacheFile(r *pb.Req) string {
+	return util.CacheName(r.Hash()) + `-rsp.json`
+}

server/src/core/fetch.go

@@ -36,7 +36,11 @@ func fetchRemote(r *pb.Req) (ab []byte, err error) {
 		return
 	}
 
-	defer rsp.Body.Close()
+	for k, v := range rsp.Header {
+		zj.J(k, v)
+	}
 
-	return io.ReadAll(rsp.Body)
+	ab, err = io.ReadAll(rsp.Body)
+	rsp.Body.Close()
+	return
 }

server/src/core/metrics.go

@@ -0,0 +1,32 @@
+package core
+
+import (
+	"encoding/json"
+	"net/http"
+	"project/metrics"
+	"project/pb"
+	"project/util"
+)
+
+func doMetrics(ab []byte, cached bool, r *http.Request) {
+
+	metrics.RspBytes(len(ab))
+
+	o := &pb.Rsp{}
+	json.Unmarshal(ab, o)
+
+	u := o.GetUsage()
+	if u == nil {
+		metrics.RspJSONFail()
+		return
+	}
+
+	metrics.RspToken(u.PromptTokens, u.TotalTokens, cached)
+
+	ip, err := util.GetIP(r)
+	sip := ip.String()
+	if err != nil {
+		sip = `unknown`
+	}
+	metrics.RspTokenByIP(sip, u.TotalTokens)
+}

server/src/core/req.go

@@ -8,12 +8,38 @@ import (
 	"project/zj"
 )
 
+func (c *Core) getAB(p *pb.Req, r *http.Request) (ab []byte, cached bool, err error) {
+	ab, ok := tryCache(p)
+	if ok {
+		cached = true
+		return
+	}
+
+	pr := c.add(p, r)
+
+	go func() {
+		reqFile := util.CacheName(p.Hash()) + `-req.json`
+		if !util.FileExists(reqFile) {
+			util.WriteFile(reqFile, p.Body)
+		}
+	}()
+
+	pr.wait()
+
+	ab = pr.rsp
+	err = pr.err
+	return
+}
+
 func req(w http.ResponseWriter, r *http.Request) (p *pb.Req, err error) {
 
 	path := r.URL.Path
 	method := r.Method
 
-	zj.J(method, path)
+	if path == `/favicon.ico` {
+		err = errSkip
+		return
+	}
 
 	ab, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1024*1024))
 	if err != nil {

server/src/core/row.go

@@ -29,6 +29,7 @@ func (pr *row) run() {
 	zj.J(`new`, s)
 
 	pr.rsp, pr.err = fetchRemote(pr.req)
+
 	go pr.saveFile()
 	go pr.metrics()
 
@@ -44,11 +45,8 @@ func (pr *row) wait() {
 }
 
 func (pr *row) saveFile() {
-	rspFile := util.CacheName(pr.req.Hash()) + `-rsp.json`
+	rspFile := rspCacheFile(pr.req)
-	if !util.FileExists(rspFile) {
+	util.WriteFile(rspFile, pr.rsp)
-		util.WriteFile(rspFile, pr.rsp)
-		zj.J(rspFile)
-	}
 }
 
 func (pr *row) metrics() {

server/src/core/web.go

@@ -1,36 +1,37 @@
 package core
 
 import (
+	"errors"
 	"net/http"
 	"project/metrics"
-	"project/util"
+	"project/zj"
 )
 
+var errSkip = errors.New(`skip`)
+
 // WebHandle ...
 func (c *Core) WebHandle(w http.ResponseWriter, r *http.Request) {
 
 	p, err := req(w, r)
 	if err != nil {
-		metrics.ReqFailCount()
+		if err != errSkip {
-		return
+			metrics.ReqFailCount()
-	}
-	metrics.ReqBytes(len(p.Body))
-
-	pr := c.add(p, r)
-
-	go func() {
-		reqFile := util.CacheName(p.Hash()) + `-req.json`
-		if !util.FileExists(reqFile) {
-			util.WriteFile(reqFile, p.Body)
 		}
-	}()
-
-	pr.wait()
-
-	if pr.err != nil {
 		err500(w)
 		return
 	}
 
-	w.Write(pr.rsp)
+	metrics.ReqBytes(len(p.Body))
+
+	ab, cached, err := c.getAB(p, r)
+	if err != nil {
+		err500(w)
+		return
+	}
+	zj.J(`cached`, cached)
+
+	w.Header().Add(`Content-Type`, `application/json`)
+	w.Write(ab)
+
+	go doMetrics(ab, cached, r)
 }

server/src/metrics/init.go

@@ -7,4 +7,11 @@ func init() {
 	prometheus.MustRegister(reqFailCount)
 	prometheus.MustRegister(reqBytes)
 	prometheus.MustRegister(errorCount)
+
+	prometheus.MustRegister(rspBytes)
+	prometheus.MustRegister(rspPromptTokenCount)
+	prometheus.MustRegister(rspTokenCount)
+	prometheus.MustRegister(rspTokenCachedCount)
+	prometheus.MustRegister(rspJSONFailCount)
+	prometheus.MustRegister(rspTokenByIP)
 }

server/src/metrics/rsp.go

@@ -0,0 +1,45 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	rspBytes            = newCounter(`orca_rsp_bytes`, `rsp bytes`)
+	rspPromptTokenCount = newCounter(`orca_rsp_prompt_token`, `prompt token`)
+	rspTokenCount       = newCounter(`orca_rsp_token`, `token`)
+	rspTokenCachedCount = newCounter(`orca_rsp_token_cached`, `token cached`)
+	rspJSONFailCount    = newCounter(`orca_rsp_json_fail`, `json fail`)
+	rspTokenByIP        = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: `orca_token_by_ip`,
+			Help: `API 返回报错`,
+		},
+		[]string{`ip`},
+	)
+)
+
+// RspToken ...
+func RspToken(prompt, total uint32, cached bool) {
+	if cached {
+		rspTokenCachedCount.Add(float64(total))
+		return
+	}
+	rspPromptTokenCount.Add(float64(prompt))
+	rspTokenCount.Add(float64(total))
+}
+
+// RspBytes ...
+func RspBytes(n int) {
+	rspBytes.Add(float64(n))
+}
+
+// RspJSONFail ...
+func RspJSONFail() {
+	rspJSONFailCount.Inc()
+}
+
+// RspTokenByIP ...
+func RspTokenByIP(ip string, token uint32) {
+	rspTokenByIP.WithLabelValues(ip).Add(float64(token))
+}

server/src/util/ip.go

@@ -28,18 +28,20 @@ func GetIP(r *http.Request) (net.IP, error) {
 		return nil, errors.New(`Invalid IP address`)
 	}
 
-	// 检查是否是IPv4
+	/*
-	parsedIPv4 := parsedIP.To4()
+		// 检查是否是IPv4
-	if parsedIPv4 == nil {
+		parsedIPv4 := parsedIP.To4()
-		return nil, errors.New(`IP address not IPv4`)
+		if parsedIPv4 == nil {
-	}
+			return nil, errors.New(`IP address not IPv4`)
+		}
 
-	// 检查是否为局域网IP
+		// 检查是否为局域网IP
-	if !parsedIP.IsPrivate() {
+		if !parsedIP.IsPrivate() {
-		return nil, errors.New(`Public IP address not allowed`)
+			return nil, errors.New(`Public IP address not allowed`)
-	}
+		}
+	*/
 
-	return parsedIPv4, nil
+	return parsedIP, nil
 }
 
 // IPString ...

server/src/web/server.go

@@ -6,6 +6,8 @@ import (
 	"project/core"
 	"project/zj"
 	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // Server ...
@@ -13,6 +15,7 @@ func Server() {
 
 	mux := http.NewServeMux()
 
+	mux.Handle(`/_metrics`, promhttp.Handler())
 	mux.HandleFunc(`/`, core.NewCore().WebHandle)
 
 	s := &http.Server{

server/src/zj/init.go

@@ -34,15 +34,13 @@ func Init() {
 
 	baseLog.SetDirPrefix(filepath.Dir(zog.GetSourceFileDir()))
 
-	// 生产环境走 docker，不写本地文件
+	dir := config.LogDir
-	if !config.Prod {
 
-		mainFile, _ := zog.NewFile(config.Dir+`/log/default.txt`, false)
+	mainFile, _ := zog.NewFile(dir+`/default.txt`, false)
-		infoFile, _ := zog.NewFile(config.Dir+`/log/io.txt`, false)
+	infoFile, _ := zog.NewFile(dir+`/io.txt`, false)
-		errFile, _ := zog.NewFile(config.Dir+`/log/err.txt`, true)
+	errFile, _ := zog.NewFile(dir+`/err.txt`, true)
 
-		mainCfg.Output = append(mainCfg.Output, mainFile)
+	mainCfg.Output = append(mainCfg.Output, mainFile)
-		infoCfg.Output = append(infoCfg.Output, infoFile)
+	infoCfg.Output = append(infoCfg.Output, infoFile)
-		errCfg.Output = append(errCfg.Output, mainFile, errFile)
+	errCfg.Output = append(errCfg.Output, mainFile, errFile)
-	}
 }