reface to openteam

2025-04-16 18:01:27 +08:00
parent bc223d6530
commit e7ffc9e8b9
92 changed files with 5345 additions and 1273 deletions
--- a/llm/aws/aws.go
+++ b/llm/aws/aws.go
@@ -0,0 +1,36 @@
+// /*
+// # AWS
+// https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-service.html
+// https://aws.amazon.com/cn/bedrock/pricing/
+// Anthropic models	Price for 1000 input tokens	Price for 1000 output tokens
+// Claude Instant      $0.00163                    $0.00551
+
+// Claude              $0.01102                    $0.03268
+
+// https://docs.aws.amazon.com/bedrock/latest/userguide/endpointsTable.html
+// 地区名称				 地区		 端点									 	  协议
+// 美国东部（弗吉尼亚北部）	美国东部1	bedrock-runtime.us-east-1.amazonaws.com		 HTTPS
+// 								   bedrock-runtime-fips.us-east-1.amazonaws.com	HTTPS
+// 美国西部（俄勒冈州）	美国西2号	 bedrock-runtime.us-west-2.amazonaws.com		 HTTPS
+// 								bedrock-runtime-fips.us-west-2.amazonaws.com	HTTPS
+// 亚太地区（新加坡）	ap-东南-1		bedrock-runtime.ap-southeast-1.amazonaws.com	HTTPS
+// */
+// //
+
+package aws
+
+// import (
+// 	"context"
+// 	"log"
+
+// 	"github.com/aws/aws-sdk-go-v2/config"
+// )
+
+// // ...
+
+// func CallClaude() {
+// 	cfg, err := config.LoadDefaultConfig(context.TODO())
+// 	if err != nil {
+// 		log.Fatalf("failed to load configuration, %v", err)
+// 	}
+// }
--- a/llm/azureopenai/azureopenai.go
+++ b/llm/azureopenai/azureopenai.go
@@ -0,0 +1,87 @@
+/*
+https://learn.microsoft.com/zh-cn/azure/cognitive-services/openai/chatgpt-quickstart
+https://learn.microsoft.com/zh-cn/azure/ai-services/openai/reference#chat-completions
+
+curl $AZURE_OPENAI_ENDPOINT/openai/deployments/gpt-35-turbo/chat/completions?api-version=2023-03-15-preview \
+  -H "Content-Type: application/json" \
+  -H "api-key: $AZURE_OPENAI_KEY" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "你好"}]
+  }'
+
+https://learn.microsoft.com/zh-cn/rest/api/cognitiveservices/azureopenaistable/models/list?tabs=HTTP
+
+  curl $AZURE_OPENAI_ENDPOINT/openai/deployments?api-version=2022-12-01 \
+  -H "Content-Type: application/json" \
+  -H "api-key: $AZURE_OPENAI_KEY" \
+
+> GPT-4 Turbo
+https://techcommunity.microsoft.com/t5/ai-azure-ai-services-blog/azure-openai-service-launches-gpt-4-turbo-and-gpt-3-5-turbo-1106/ba-p/3985962
+
+*/
+
+package azureopenai
+
+import (
+	"encoding/json"
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+var (
+	ENDPOINT        string
+	API_KEY         string
+	DEPLOYMENT_NAME string
+)
+
+type ModelsList struct {
+	Data []struct {
+		ScaleSettings struct {
+			ScaleType string `json:"scale_type"`
+		} `json:"scale_settings"`
+		Model     string `json:"model"`
+		Owner     string `json:"owner"`
+		ID        string `json:"id"`
+		Status    string `json:"status"`
+		CreatedAt int    `json:"created_at"`
+		UpdatedAt int    `json:"updated_at"`
+		Object    string `json:"object"`
+	} `json:"data"`
+	Object string `json:"object"`
+}
+
+func Models(endpoint, apikey string) (*ModelsList, error) {
+	endpoint = RemoveTrailingSlash(endpoint)
+	var modelsl ModelsList
+	req, _ := http.NewRequest(http.MethodGet, endpoint+"/openai/deployments?api-version=2022-12-01", nil)
+	req.Header.Set("api-key", apikey)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	err = json.NewDecoder(resp.Body).Decode(&modelsl)
+	if err != nil {
+		return nil, err
+	}
+	return &modelsl, nil
+}
+
+func RemoveTrailingSlash(s string) string {
+	const prefix = "openai.azure.com/"
+	if strings.HasSuffix(strings.TrimSpace(s), prefix) && strings.HasSuffix(s, "/") {
+		return s[:len(s)-1]
+	}
+	return s
+}
+
+func GetResourceName(url string) string {
+	re := regexp.MustCompile(`https?://(.+)\.openai\.azure\.com/?`)
+	match := re.FindStringSubmatch(url)
+	if len(match) > 1 {
+		return match[1]
+	}
+	return ""
+}
--- a/llm/claude/chat.go
+++ b/llm/claude/chat.go
@@ -0,0 +1,138 @@
+// https://docs.anthropic.com/claude/reference/messages_post
+
+package claude
+
+import (
+	"context"
+	"encoding/json"
+	"opencatd-open/internal/model"
+	"opencatd-open/llm"
+	"opencatd-open/llm/openai"
+
+	"github.com/gin-gonic/gin"
+)
+
+func ChatProxy(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
+	ChatMessages(c, chatReq)
+}
+
+func ChatTextCompletions(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
+
+}
+
+type ChatRequest struct {
+	Model            string  `json:"model,omitempty"`
+	Messages         any     `json:"messages,omitempty"`
+	MaxTokens        int     `json:"max_tokens,omitempty"`
+	Stream           bool    `json:"stream,omitempty"`
+	System           string  `json:"system,omitempty"`
+	TopK             int     `json:"top_k,omitempty"`
+	TopP             float64 `json:"top_p,omitempty"`
+	Temperature      float64 `json:"temperature,omitempty"`
+	AnthropicVersion string  `json:"anthropic_version,omitempty"`
+}
+
+func (c *ChatRequest) ByteJson() []byte {
+	bytejson, _ := json.Marshal(c)
+	return bytejson
+}
+
+type ChatMessage struct {
+	Role    string `json:"role,omitempty"`
+	Content string `json:"content,omitempty"`
+}
+
+type VisionMessages struct {
+	Role    string          `json:"role,omitempty"`
+	Content []VisionContent `json:"content,omitempty"`
+}
+
+type VisionContent struct {
+	Type   string        `json:"type,omitempty"`
+	Source *VisionSource `json:"source,omitempty"`
+	Text   string        `json:"text,omitempty"`
+}
+
+type VisionSource struct {
+	Type      string `json:"type,omitempty"`
+	MediaType string `json:"media_type,omitempty"`
+	Data      string `json:"data,omitempty"`
+}
+
+type ChatResponse struct {
+	ID           string `json:"id"`
+	Type         string `json:"type"`
+	Role         string `json:"role"`
+	Model        string `json:"model"`
+	StopSequence any    `json:"stop_sequence"`
+	Usage        struct {
+		InputTokens  int `json:"input_tokens"`
+		OutputTokens int `json:"output_tokens"`
+	} `json:"usage"`
+	Content []struct {
+		Type string `json:"type"`
+		Text string `json:"text"`
+	} `json:"content"`
+	StopReason string `json:"stop_reason"`
+}
+
+type ClaudeStreamResponse struct {
+	Type         string `json:"type"`
+	Index        int    `json:"index"`
+	ContentBlock struct {
+		Type string `json:"type"`
+		Text string `json:"text"`
+	} `json:"content_block"`
+	Delta struct {
+		Type         string `json:"type"`
+		Text         string `json:"text"`
+		StopReason   string `json:"stop_reason"`
+		StopSequence any    `json:"stop_sequence"`
+	} `json:"delta"`
+	Message struct {
+		ID           string `json:"id"`
+		Type         string `json:"type"`
+		Role         string `json:"role"`
+		Content      []any  `json:"content"`
+		Model        string `json:"model"`
+		StopReason   string `json:"stop_reason"`
+		StopSequence any    `json:"stop_sequence"`
+		Usage        struct {
+			InputTokens  int `json:"input_tokens"`
+			OutputTokens int `json:"output_tokens"`
+		} `json:"usage"`
+	} `json:"message"`
+	Error struct {
+		Type    string `json:"type"`
+		Message string `json:"message"`
+	} `json:"error"`
+	Usage struct {
+		OutputTokens int `json:"output_tokens"`
+	} `json:"usage"`
+}
+
+type Claude struct {
+	Ctx context.Context
+
+	ApiKey     *model.ApiKey
+	tokenUsage *llm.TokenUsage
+
+	Done chan struct{}
+}
+
+func NewClaude(ctx context.Context, apiKey *model.ApiKey) (*Claude, error) {
+	return &Claude{
+		Ctx:        context.Background(),
+		ApiKey:     apiKey,
+		tokenUsage: &llm.TokenUsage{},
+		Done:       make(chan struct{}),
+	}, nil
+}
+
+func (c *Claude) Chat(ctx context.Context, chatReq llm.ChatRequest) (*llm.ChatResponse, error) {
+	return nil, nil
+}
+
+func (g *Claude) StreamChat(ctx context.Context, chatReq llm.ChatRequest) (chan *llm.StreamChatResponse, error) {
+	return nil, nil
+}
--- a/llm/claude/claude.go
+++ b/llm/claude/claude.go
@@ -0,0 +1,429 @@
+/*
+https://docs.anthropic.com/claude/reference/complete_post
+
+curl --request POST \
+     --url https://api.anthropic.com/v1/complete \
+     --header "anthropic-version: 2023-06-01" \
+     --header "content-type: application/json" \
+     --header "x-api-key: $ANTHROPIC_API_KEY" \
+     --data '
+{
+  "model": "claude-2",
+  "prompt": "\n\nHuman: Hello, world!\n\nAssistant:",
+  "max_tokens_to_sample": 256,
+  "stream": true
+}
+'
+
+{"completion":" Hello! Nice to meet you.","stop_reason":"stop_sequence","model":"claude-2.0","stop":"\n\nHuman:","log_id":"727bded01002627057967d02b3d557a01aa73266849b62f5aa0b97dec1247ed3"}
+
+event: completion
+data: {"completion":"","stop_reason":"stop_sequence","model":"claude-2.0","stop":"\n\nHuman:","log_id":"dfd42341ad08856ff01811885fb8640a1bf977551d8331f81fe9a6c8182c6c63"}
+
+# Model Pricing
+
+Claude Instant |100,000 tokens |Prompt $1.63/million tokens  |Completion $5.51/million tokens
+
+Claude 2       |100,000 tokens |Prompt $11.02/million tokens |Completion $32.68/million tokens
+*Claude 1 is still accessible and offered at the same price as Claude 2.
+
+# AWS
+https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-service.html
+https://aws.amazon.com/cn/bedrock/pricing/
+Anthropic models	Price for 1000 input tokens	Price for 1000 output tokens
+Claude Instant      $0.00163                    $0.00551
+
+Claude              $0.01102                    $0.03268
+
+https://docs.aws.amazon.com/bedrock/latest/userguide/endpointsTable.html
+地区名称				 地区		 端点									 	  协议
+美国东部（弗吉尼亚北部）	美国东部1	bedrock-runtime.us-east-1.amazonaws.com		 HTTPS
+								   bedrock-runtime-fips.us-east-1.amazonaws.com	HTTPS
+美国西部（俄勒冈州）	美国西2号	 bedrock-runtime.us-west-2.amazonaws.com		 HTTPS
+								bedrock-runtime-fips.us-west-2.amazonaws.com	HTTPS
+亚太地区（新加坡）	ap-东南-1		bedrock-runtime.ap-southeast-1.amazonaws.com	HTTPS
+*/
+
+// package anthropic
+package claude
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sashabaranov/go-openai"
+)
+
+var (
+	ClaudeUrl             = "https://api.anthropic.com/v1/complete"
+	ClaudeMessageEndpoint = "https://api.anthropic.com/v1/messages"
+)
+
+type MessageModule struct {
+	Assistant string // returned data (do not modify)
+	Human     string // input content
+}
+
+type CompleteRequest struct {
+	Model             string `json:"model,omitempty"`                //*
+	Prompt            string `json:"prompt,omitempty"`               //*
+	MaxTokensToSample int    `json:"max_tokens_to_sample,omitempty"` //*
+	StopSequences     string `json:"stop_sequences,omitempty"`
+	Temperature       int    `json:"temperature,omitempty"`
+	TopP              int    `json:"top_p,omitempty"`
+	TopK              int    `json:"top_k,omitempty"`
+	Stream            bool   `json:"stream,omitempty"`
+	Metadata          struct {
+		UserId string `json:"user_Id,omitempty"`
+	} `json:"metadata,omitempty"`
+}
+
+type CompleteResponse struct {
+	Completion string `json:"completion"`
+	StopReason string `json:"stop_reason"`
+	Model      string `json:"model"`
+	Stop       string `json:"stop"`
+	LogID      string `json:"log_id"`
+}
+
+func Create() {
+	complet := CompleteRequest{
+		Model:  "claude-2",
+		Prompt: "Human: Hello, world!\\n\\nAssistant:",
+		Stream: true,
+	}
+	var payload *bytes.Buffer
+	json.NewEncoder(payload).Encode(complet)
+
+	// payload := strings.NewReader("{\"model\":\"claude-2\",\"prompt\":\"\\n\\nHuman: Hello, world!\\n\\nAssistant:\",\"max_tokens_to_sample\":256}")
+
+	req, _ := http.NewRequest("POST", ClaudeUrl, payload)
+
+	req.Header.Add("accept", "application/json")
+	req.Header.Add("anthropic-version", "2023-06-01")
+	req.Header.Add("x-api-key", "$ANTHROPIC_API_KEY")
+	req.Header.Add("content-type", "application/json")
+
+	res, _ := http.DefaultClient.Do(req)
+
+	defer res.Body.Close()
+	// body, _ := io.ReadAll(res.Body)
+
+	// fmt.Println(string(body))
+	reader := bufio.NewReader(res.Body)
+	for {
+		line, err := reader.ReadString('\n')
+		if err == nil {
+			if strings.HasPrefix(line, "data:") {
+				fmt.Println(line)
+				// var result CompleteResponse
+				// json.Unmarshal()
+			} else {
+				continue
+			}
+		} else {
+			break
+		}
+	}
+}
+
+func ClaudeProxy(c *gin.Context) {
+	var chatlog store.Tokens
+	var complete CompleteRequest
+
+	byteBody, _ := io.ReadAll(c.Request.Body)
+	c.Request.Body = io.NopCloser(bytes.NewBuffer(byteBody))
+
+	if err := json.Unmarshal(byteBody, &complete); err != nil {
+		c.AbortWithError(http.StatusBadRequest, err)
+		return
+	}
+
+	key, err := store.SelectKeyCache("claude") //anthropic
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+
+	chatlog.Model = complete.Model
+
+	token, _ := c.Get("localuser")
+
+	lu, err := store.GetUserByToken(token.(string))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+	chatlog.UserID = int(lu.ID)
+
+	chatlog.PromptCount = tokenizer.NumTokensFromStr(complete.Prompt, complete.Model)
+
+	if key.EndPoint == "" {
+		key.EndPoint = "https://api.anthropic.com"
+	}
+	targetUrl, _ := url.ParseRequestURI(key.EndPoint + c.Request.URL.String())
+
+	proxy := httputil.NewSingleHostReverseProxy(targetUrl)
+	proxy.Director = func(req *http.Request) {
+		req.Host = targetUrl.Host
+		req.URL.Scheme = targetUrl.Scheme
+		req.URL.Host = targetUrl.Host
+
+		req.Header.Set("anthropic-version", "2023-06-01")
+		req.Header.Set("content-type", "application/json")
+		req.Header.Set("x-api-key", key.Key)
+	}
+
+	proxy.ModifyResponse = func(resp *http.Response) error {
+		if resp.StatusCode != http.StatusOK {
+			return nil
+		}
+		var byteResp []byte
+		byteResp, _ = io.ReadAll(resp.Body)
+		resp.Body = io.NopCloser(bytes.NewBuffer(byteResp))
+		if complete.Stream != true {
+			var complete_resp CompleteResponse
+
+			if err := json.Unmarshal(byteResp, &complete_resp); err != nil {
+				log.Println(err)
+				return nil
+			}
+			chatlog.CompletionCount = tokenizer.NumTokensFromStr(complete_resp.Completion, chatlog.Model)
+		} else {
+			var completion string
+			for {
+				line, err := bufio.NewReader(bytes.NewBuffer(byteResp)).ReadString('\n')
+				if err != nil {
+					if strings.HasPrefix(line, "data:") {
+						line = strings.TrimSpace(strings.TrimPrefix(line, "data:"))
+						if strings.HasSuffix(line, "[DONE]") {
+							break
+						}
+						line = strings.TrimSpace(line)
+						var complete_resp CompleteResponse
+						if err := json.Unmarshal([]byte(line), &complete_resp); err != nil {
+							log.Println(err)
+							break
+						}
+						completion += line
+					}
+				}
+			}
+			log.Println("completion:", completion)
+			chatlog.CompletionCount = tokenizer.NumTokensFromStr(completion, chatlog.Model)
+		}
+
+		// calc cost
+		chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
+		chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
+
+		if err := store.Record(&chatlog); err != nil {
+			log.Println(err)
+		}
+		if err := store.SumDaily(chatlog.UserID); err != nil {
+			log.Println(err)
+		}
+		return nil
+	}
+	proxy.ServeHTTP(c.Writer, c.Request)
+}
+
+func TransReq(chatreq *openai.ChatCompletionRequest) (*bytes.Buffer, error) {
+	transReq := CompleteRequest{
+		Model:             chatreq.Model,
+		Temperature:       int(chatreq.Temperature),
+		TopP:              int(chatreq.TopP),
+		Stream:            chatreq.Stream,
+		MaxTokensToSample: chatreq.MaxTokens,
+	}
+	if transReq.MaxTokensToSample == 0 {
+		transReq.MaxTokensToSample = 100000
+	}
+	var prompt string
+	for _, msg := range chatreq.Messages {
+		switch msg.Role {
+		case "system":
+			prompt += fmt.Sprintf("\n\nHuman:%s", msg.Content)
+		case "user":
+			prompt += fmt.Sprintf("\n\nHuman:%s", msg.Content)
+		case "assistant":
+			prompt += fmt.Sprintf("\n\nAssistant:%s", msg.Content)
+		}
+	}
+	transReq.Prompt = prompt + "\n\nAssistant:"
+	var payload = bytes.NewBuffer(nil)
+	if err := json.NewEncoder(payload).Encode(transReq); err != nil {
+		return nil, err
+	}
+	return payload, nil
+}
+
+func TransRsp(c *gin.Context, isStream bool, chatlog store.Tokens, reader *bufio.Reader) {
+	if !isStream {
+		var completersp CompleteResponse
+		var chatrsp openai.ChatCompletionResponse
+		json.NewDecoder(reader).Decode(&completersp)
+		chatrsp.Model = completersp.Model
+		chatrsp.ID = completersp.LogID
+		chatrsp.Object = "chat.completion"
+		chatrsp.Created = time.Now().Unix()
+		choice := openai.ChatCompletionChoice{
+			Index:        0,
+			FinishReason: "stop",
+			Message: openai.ChatCompletionMessage{
+				Role:    "assistant",
+				Content: completersp.Completion,
+			},
+		}
+		chatrsp.Choices = append(chatrsp.Choices, choice)
+		var payload *bytes.Buffer
+		if err := json.NewEncoder(payload).Encode(chatrsp); err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error": gin.H{
+					"message": err.Error(),
+				},
+			})
+			return
+		}
+		chatlog.CompletionCount = tokenizer.NumTokensFromStr(completersp.Completion, chatlog.Model)
+		chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
+		chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
+		if err := store.Record(&chatlog); err != nil {
+			log.Println(err)
+		}
+		if err := store.SumDaily(chatlog.UserID); err != nil {
+			log.Println(err)
+		}
+
+		c.JSON(http.StatusOK, payload)
+		return
+	} else {
+		var (
+			wg            sync.WaitGroup
+			dataChan      = make(chan string)
+			stopChan      = make(chan bool)
+			complete_resp string
+		)
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			for {
+				line, err := reader.ReadString('\n')
+				if err == nil {
+					if strings.HasPrefix(line, "data: ") {
+						var result CompleteResponse
+						json.NewDecoder(strings.NewReader(line[6:])).Decode(&result)
+						if result.StopReason == "" {
+							if result.Completion != "" {
+								complete_resp += result.Completion
+								chatrsp := openai.ChatCompletionStreamResponse{
+									ID:      result.LogID,
+									Model:   result.Model,
+									Object:  "chat.completion",
+									Created: time.Now().Unix(),
+								}
+								choice := openai.ChatCompletionStreamChoice{
+									Delta: openai.ChatCompletionStreamChoiceDelta{
+										Role:    "assistant",
+										Content: result.Completion,
+									},
+									FinishReason: "",
+								}
+								chatrsp.Choices = append(chatrsp.Choices, choice)
+								bytedate, _ := json.Marshal(chatrsp)
+								dataChan <- string(bytedate)
+							}
+						} else {
+							chatrsp := openai.ChatCompletionStreamResponse{
+								ID:      result.LogID,
+								Model:   result.Model,
+								Object:  "chat.completion",
+								Created: time.Now().Unix(),
+							}
+							choice := openai.ChatCompletionStreamChoice{
+								Delta: openai.ChatCompletionStreamChoiceDelta{
+									Role:    "assistant",
+									Content: result.Completion,
+								},
+							}
+							choice.FinishReason = openai.FinishReason(TranslatestopReason(result.StopReason))
+							chatrsp.Choices = append(chatrsp.Choices, choice)
+							bytedate, _ := json.Marshal(chatrsp)
+							dataChan <- string(bytedate)
+							dataChan <- "[DONE]"
+							break
+						}
+					} else {
+						continue
+					}
+				} else {
+					break
+				}
+			}
+
+			close(dataChan)
+			stopChan <- true
+			close(stopChan)
+		}()
+
+		go func() {
+			defer wg.Done()
+		Loop:
+			for {
+				select {
+				case data := <-dataChan:
+					if data != "" {
+						c.Writer.WriteString("data: " + data)
+						c.Writer.WriteString("\n\n")
+						c.Writer.Flush()
+					}
+				case <-stopChan:
+					break Loop
+				}
+			}
+		}()
+		wg.Wait()
+		chatlog.CompletionCount = tokenizer.NumTokensFromStr(complete_resp, chatlog.Model)
+		chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
+		chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
+		if err := store.Record(&chatlog); err != nil {
+			log.Println(err)
+		}
+		if err := store.SumDaily(chatlog.UserID); err != nil {
+			log.Println(err)
+		}
+	}
+}
+
+// claude -> openai
+func TranslatestopReason(reason string) string {
+	switch reason {
+	case "stop_sequence":
+		return "stop"
+	case "max_tokens":
+		return "length"
+	default:
+		return reason
+	}
+}
--- a/llm/claude/handle_proxy.go
+++ b/llm/claude/handle_proxy.go
@@ -0,0 +1,230 @@
+package claude
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"opencatd-open/llm/openai"
+	"opencatd-open/llm/vertexai"
+	"opencatd-open/pkg/error"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+func ChatMessages(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
+	var (
+		req       *http.Request
+		targetURL = ClaudeMessageEndpoint
+	)
+
+	apiKey, err := store.SelectKeyCache("claude")
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	usagelog := store.Tokens{Model: chatReq.Model}
+	var claudReq ChatRequest
+	claudReq.Model = chatReq.Model
+	claudReq.Stream = chatReq.Stream
+	// claudReq.Temperature = chatReq.Temperature
+	claudReq.TopP = chatReq.TopP
+	claudReq.MaxTokens = 4096
+	if apiKey.ApiType == "vertex" {
+		claudReq.AnthropicVersion = "vertex-2023-10-16"
+		claudReq.Model = ""
+	}
+
+	var claudecontent []VisionContent
+	var prompt string
+	for _, msg := range chatReq.Messages {
+		switch ct := msg.Content.(type) {
+		case string:
+			prompt += "<" + msg.Role + ">: " + msg.Content.(string) + "\n"
+			if msg.Role == "system" {
+				claudReq.System = msg.Content.(string)
+				continue
+			}
+			claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + msg.Content.(string)})
+		case []any:
+			for _, item := range ct {
+				if m, ok := item.(map[string]interface{}); ok {
+					if m["type"] == "text" {
+						prompt += "<" + msg.Role + ">: " + m["text"].(string) + "\n"
+						claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + m["text"].(string)})
+					} else if m["type"] == "image_url" {
+						if url, ok := m["image_url"].(map[string]interface{}); ok {
+							fmt.Printf("  URL: %v\n", url["url"])
+							if strings.HasPrefix(url["url"].(string), "http") {
+								fmt.Println("网络图片:", url["url"].(string))
+							} else if strings.HasPrefix(url["url"].(string), "data:image") {
+								fmt.Println("base64:", url["url"].(string)[:20])
+								var mediaType string
+								if strings.HasPrefix(url["url"].(string), "data:image/jpeg") {
+									mediaType = "image/jpeg"
+								}
+								if strings.HasPrefix(url["url"].(string), "data:image/png") {
+									mediaType = "image/png"
+								}
+								claudecontent = append(claudecontent, VisionContent{Type: "image", Source: &VisionSource{Type: "base64", MediaType: mediaType, Data: strings.Split(url["url"].(string), ",")[1]}})
+							}
+						}
+					}
+				}
+			}
+		default:
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error": gin.H{
+					"message": "Invalid content type",
+				},
+			})
+			return
+		}
+		if len(chatReq.Tools) > 0 {
+			tooljson, _ := json.Marshal(chatReq.Tools)
+			prompt += "<tools>: " + string(tooljson) + "\n"
+		}
+	}
+
+	claudReq.Messages = []VisionMessages{{Role: "user", Content: claudecontent}}
+
+	usagelog.PromptCount = tokenizer.NumTokensFromStr(prompt, chatReq.Model)
+
+	if apiKey.ApiType == "vertex" {
+		var vertexSecret vertexai.VertexSecretKey
+		if err := json.Unmarshal([]byte(apiKey.ApiSecret), &vertexSecret); err != nil {
+			c.JSON(http.StatusInternalServerError, error.ErrorData(err.Error()))
+			return
+		}
+
+		vcmodel, ok := vertexai.VertexClaudeModelMap[chatReq.Model]
+		if !ok {
+			c.JSON(http.StatusInternalServerError, error.ErrorData("Model not found"))
+			return
+		}
+
+		// 获取gcloud token，临时放置在apiKey.Key中
+		gcloudToken, err := vertexai.GcloudAuth(vertexSecret.ClientEmail, vertexSecret.PrivateKey)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, error.ErrorData(err.Error()))
+			return
+		}
+
+		// 拼接vertex的请求地址
+		targetURL = fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1/projects/%s/locations/%s/publishers/anthropic/models/%s:streamRawPredict", vcmodel.Region, vertexSecret.ProjectID, vcmodel.Region, vcmodel.VertexName)
+
+		req, _ = http.NewRequest("POST", targetURL, bytes.NewReader(claudReq.ByteJson()))
+		req.Header.Set("Authorization", "Bearer "+gcloudToken)
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Accept", "text/event-stream")
+		req.Header.Set("Accept-Encoding", "identity")
+	} else {
+		req, _ = http.NewRequest("POST", targetURL, bytes.NewReader(claudReq.ByteJson()))
+		req.Header.Set("x-api-key", apiKey.Key)
+		req.Header.Set("anthropic-version", "2023-06-01")
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	client := http.DefaultClient
+	rsp, err := client.Do(req)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	defer rsp.Body.Close()
+	if rsp.StatusCode != http.StatusOK {
+		io.Copy(c.Writer, rsp.Body)
+		return
+	}
+	var buffer bytes.Buffer
+	teeReader := io.TeeReader(rsp.Body, &buffer)
+
+	dataChan := make(chan string)
+	// stopChan := make(chan bool)
+
+	var result string
+
+	scanner := bufio.NewScanner(teeReader)
+
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if len(line) > 0 && bytes.HasPrefix(line, []byte("data: ")) {
+				if bytes.HasPrefix(line, []byte("data: [DONE]")) {
+					dataChan <- string(line) + "\n"
+					break
+				}
+				var claudeResp ClaudeStreamResponse
+				line = bytes.Replace(line, []byte("data: "), []byte(""), -1)
+				line = bytes.TrimSpace(line)
+				if err := json.Unmarshal(line, &claudeResp); err != nil {
+					continue
+				}
+
+				if claudeResp.Type == "message_start" {
+					if claudeResp.Message.Role != "" {
+						result += "<" + claudeResp.Message.Role + ">"
+					}
+				} else if claudeResp.Type == "message_stop" {
+					break
+				}
+
+				if claudeResp.Delta.Text != "" {
+					result += claudeResp.Delta.Text
+				}
+				var choice openai.Choice
+				choice.Delta.Role = claudeResp.Message.Role
+				choice.Delta.Content = claudeResp.Delta.Text
+				choice.FinishReason = claudeResp.Delta.StopReason
+
+				chatResp := openai.ChatCompletionStreamResponse{
+					Model:   chatReq.Model,
+					Choices: []openai.Choice{choice},
+				}
+				dataChan <- "data: " + string(chatResp.ByteJson()) + "\n"
+				if claudeResp.Delta.StopReason != "" {
+					dataChan <- "\ndata: [DONE]\n"
+				}
+			}
+		}
+		defer close(dataChan)
+	}()
+
+	c.Writer.Header().Set("Content-Type", "text/event-stream")
+	c.Writer.Header().Set("Cache-Control", "no-cache")
+	c.Writer.Header().Set("Connection", "keep-alive")
+	c.Writer.Header().Set("Transfer-Encoding", "chunked")
+	c.Writer.Header().Set("X-Accel-Buffering", "no")
+
+	c.Stream(func(w io.Writer) bool {
+		if data, ok := <-dataChan; ok {
+			if strings.HasPrefix(data, "data: ") {
+				c.Writer.WriteString(data)
+				// c.Writer.WriteString("\n\n")
+			} else {
+				c.Writer.WriteHeader(http.StatusBadGateway)
+				c.Writer.WriteString(data)
+			}
+			c.Writer.Flush()
+			return true
+		}
+		go func() {
+			usagelog.CompletionCount = tokenizer.NumTokensFromStr(result, chatReq.Model)
+			usagelog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(usagelog.Model, usagelog.PromptCount, usagelog.CompletionCount))
+			if err := store.Record(&usagelog); err != nil {
+				log.Println(err)
+			}
+			if err := store.SumDaily(usagelog.UserID); err != nil {
+				log.Println(err)
+			}
+		}()
+		return false
+	})
+}
--- a/llm/claude/v2/chat.go
+++ b/llm/claude/v2/chat.go
@@ -0,0 +1,246 @@
+package claude
+
+import (
+	"context"
+	"encoding/base64"
+	"net/http"
+	"net/url"
+	"opencatd-open/internal/model"
+	"opencatd-open/llm"
+	"os"
+	"strings"
+
+	"github.com/liushuangls/go-anthropic/v2"
+	"github.com/sashabaranov/go-openai"
+)
+
+type Claude struct {
+	Ctx        context.Context
+	ApiKey     *model.ApiKey
+	tokenUsage *llm.TokenUsage
+	Done       chan struct{}
+	Client     *anthropic.Client
+}
+
+func NewClaude(apiKey *model.ApiKey) (*Claude, error) {
+	opts := []anthropic.ClientOption{}
+	if os.Getenv("LOCAL_PROXY") != "" {
+		proxyUrl, err := url.Parse(os.Getenv("LOCAL_PROXY"))
+		if err == nil {
+			client := http.DefaultClient
+			client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
+			opts = append(opts, anthropic.WithHTTPClient(client))
+		}
+	}
+	return &Claude{
+		Ctx:        context.Background(),
+		ApiKey:     apiKey,
+		tokenUsage: &llm.TokenUsage{},
+		Done:       make(chan struct{}),
+		Client:     anthropic.NewClient(*apiKey.ApiKey, opts...),
+	}, nil
+}
+
+func (c *Claude) Chat(ctx context.Context, chatReq llm.ChatRequest) (*llm.ChatResponse, error) {
+	var messages []anthropic.Message
+
+	if len(chatReq.Messages) > 0 {
+		for _, msg := range chatReq.Messages {
+			var role anthropic.ChatRole
+			if msg.Role != "assistant" {
+				role = anthropic.RoleUser
+			} else {
+				role = anthropic.RoleAssistant
+			}
+
+			var content []anthropic.MessageContent
+			if len(msg.MultiContent) > 0 {
+				for _, mc := range msg.MultiContent {
+					if mc.Type == "text" {
+						content = append(content, anthropic.MessageContent{Type: anthropic.MessagesContentTypeText, Text: &mc.Text})
+					}
+					if mc.Type == "image_url" {
+						if strings.HasPrefix(mc.ImageURL.URL, "http") {
+							continue
+						}
+						if strings.HasPrefix(mc.ImageURL.URL, "data:image") {
+							var mediaType string
+							if strings.HasPrefix(mc.ImageURL.URL, "data:image/jpeg") {
+								mediaType = "image/jpeg"
+							}
+							if strings.HasPrefix(mc.ImageURL.URL, "data:image/png") {
+								mediaType = "image/png"
+							}
+							imageString := strings.Split(mc.ImageURL.URL, ",")[1]
+							imageBytes, _ := base64.StdEncoding.DecodeString(imageString)
+
+							content = append(content, anthropic.MessageContent{Type: "image", Source: &anthropic.MessageContentSource{Type: "base64", MediaType: mediaType, Data: imageBytes}})
+						}
+
+					}
+					messages = append(messages, anthropic.Message{Role: role, Content: content})
+				}
+			} else {
+				if len(msg.Content) > 0 {
+					content = append(content, anthropic.MessageContent{Type: "text", Text: &msg.Content})
+				}
+			}
+			messages = append(messages, anthropic.Message{Role: role, Content: content})
+		}
+	}
+
+	var maxTokens int
+	if chatReq.MaxTokens > 0 {
+		maxTokens = chatReq.MaxTokens
+	} else {
+		if strings.Contains(chatReq.Model, "sonnet") || strings.Contains(chatReq.Model, "haiku") {
+			maxTokens = 8192
+		} else {
+			maxTokens = 4096
+		}
+	}
+
+	resp, err := c.Client.CreateMessages(ctx, anthropic.MessagesRequest{
+		Model:     anthropic.Model(chatReq.Model),
+		Messages:  messages,
+		MaxTokens: maxTokens,
+		Stream:    false,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	c.tokenUsage.PromptTokens += resp.Usage.InputTokens
+	c.tokenUsage.CompletionTokens += resp.Usage.OutputTokens
+	c.tokenUsage.TotalTokens += resp.Usage.InputTokens + resp.Usage.OutputTokens
+
+	return &llm.ChatResponse{
+		Model: string(resp.Model),
+		Choices: []openai.ChatCompletionChoice{
+			{
+				FinishReason: openai.FinishReason(resp.StopReason),
+				Message: openai.ChatCompletionMessage{
+					Role:    openai.ChatMessageRoleAssistant,
+					Content: *resp.Content[0].Text,
+				},
+			},
+		},
+	}, nil
+}
+
+func (c *Claude) StreamChat(ctx context.Context, chatReq llm.ChatRequest) (chan *llm.StreamChatResponse, error) {
+	var messages []anthropic.Message
+
+	if len(chatReq.Messages) > 0 {
+		for _, msg := range chatReq.Messages {
+			var role anthropic.ChatRole
+			if msg.Role != "assistant" {
+				role = anthropic.RoleUser
+			} else {
+				role = anthropic.RoleAssistant
+			}
+
+			var content []anthropic.MessageContent
+			if len(msg.MultiContent) > 0 {
+				for _, mc := range msg.MultiContent {
+					if mc.Type == "text" {
+						content = append(content, anthropic.MessageContent{Type: anthropic.MessagesContentTypeText, Text: &mc.Text})
+					}
+					if mc.Type == "image_url" {
+						if strings.HasPrefix(mc.ImageURL.URL, "http") {
+							continue
+						}
+						if strings.HasPrefix(mc.ImageURL.URL, "data:image") {
+							var mediaType string
+							if strings.HasPrefix(mc.ImageURL.URL, "data:image/jpeg") {
+								mediaType = "image/jpeg"
+							}
+							if strings.HasPrefix(mc.ImageURL.URL, "data:image/png") {
+								mediaType = "image/png"
+							}
+							imageString := strings.Split(mc.ImageURL.URL, ",")[1]
+							imageBytes, _ := base64.StdEncoding.DecodeString(imageString)
+
+							content = append(content, anthropic.MessageContent{Type: "image", Source: &anthropic.MessageContentSource{Type: "base64", MediaType: mediaType, Data: imageBytes}})
+						}
+
+					}
+					messages = append(messages, anthropic.Message{Role: role, Content: content})
+				}
+			} else {
+				if len(msg.Content) > 0 {
+					content = append(content, anthropic.MessageContent{Type: "text", Text: &msg.Content})
+				}
+			}
+			messages = append(messages, anthropic.Message{Role: role, Content: content})
+		}
+	}
+
+	var maxTokens int
+	if chatReq.MaxTokens > 0 {
+		maxTokens = chatReq.MaxTokens
+	} else {
+		if strings.Contains(chatReq.Model, "sonnet") || strings.Contains(chatReq.Model, "haiku") {
+			maxTokens = 8192
+		} else {
+			maxTokens = 4096
+		}
+	}
+
+	datachan := make(chan *llm.StreamChatResponse)
+	// var resp anthropic.MessagesResponse
+	var err error
+	go func() {
+		defer close(datachan)
+		_, err = c.Client.CreateMessagesStream(ctx, anthropic.MessagesStreamRequest{
+			MessagesRequest: anthropic.MessagesRequest{
+				Model:     anthropic.Model(chatReq.Model),
+				Messages:  messages,
+				MaxTokens: maxTokens,
+			},
+			OnContentBlockDelta: func(data anthropic.MessagesEventContentBlockDeltaData) {
+				datachan <- &llm.StreamChatResponse{
+					Model: chatReq.Model,
+					Choices: []openai.ChatCompletionStreamChoice{
+						{
+							Delta: openai.ChatCompletionStreamChoiceDelta{Content: *data.Delta.Text},
+						},
+					},
+				}
+			},
+			OnMessageStart: func(memss anthropic.MessagesEventMessageStartData) {
+				c.tokenUsage.PromptTokens += memss.Message.Usage.InputTokens
+				c.tokenUsage.CompletionTokens += memss.Message.Usage.OutputTokens
+				c.tokenUsage.TotalTokens += memss.Message.Usage.InputTokens + memss.Message.Usage.OutputTokens
+			},
+			OnMessageDelta: func(memdd anthropic.MessagesEventMessageDeltaData) {
+				c.tokenUsage.PromptTokens += memdd.Usage.InputTokens
+				c.tokenUsage.CompletionTokens += memdd.Usage.OutputTokens
+				c.tokenUsage.TotalTokens += memdd.Usage.InputTokens + memdd.Usage.OutputTokens
+
+				datachan <- &llm.StreamChatResponse{
+					Model: chatReq.Model,
+					Choices: []openai.ChatCompletionStreamChoice{
+						{FinishReason: openai.FinishReason(memdd.Delta.StopReason)},
+					},
+				}
+			},
+		})
+
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+	}()
+	if err != nil {
+		return nil, err
+	}
+
+	return datachan, err
+}
+
+func (c *Claude) GetTokenUsage() *llm.TokenUsage {
+	return c.tokenUsage
+
+}
--- a/llm/google/chat.go
+++ b/llm/google/chat.go
@@ -0,0 +1,244 @@
+// https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/grounding-search-entry-points?authuser=2&hl=zh-cn
+//
+//	https://cloud.google.com/vertex-ai/docs/generative-ai/quotas-genai
+package google
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"opencatd-open/llm/openai"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/google/generative-ai-go/genai"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+)
+
+type GeminiChatRequest struct {
+	Contents []GeminiContent `json:"contents,omitempty"`
+}
+
+func (g GeminiChatRequest) ByteJson() []byte {
+	bytejson, _ := json.Marshal(g)
+	return bytejson
+}
+
+type GeminiContent struct {
+	Role  string       `json:"role,omitempty"`
+	Parts []GeminiPart `json:"parts,omitempty"`
+}
+type GeminiPart struct {
+	Text string `json:"text,omitempty"`
+	// InlineData GeminiPartInlineData `json:"inlineData,omitempty"`
+}
+type GeminiPartInlineData struct {
+	MimeType string `json:"mimeType,omitempty"`
+	Data     string `json:"data,omitempty"` // base64
+}
+
+type GeminiResponse struct {
+	Candidates []struct {
+		Content struct {
+			Parts []struct {
+				Text string `json:"text"`
+			} `json:"parts"`
+			Role string `json:"role"`
+		} `json:"content"`
+		FinishReason  string `json:"finishReason"`
+		Index         int    `json:"index"`
+		SafetyRatings []struct {
+			Category    string `json:"category"`
+			Probability string `json:"probability"`
+		} `json:"safetyRatings"`
+	} `json:"candidates"`
+	PromptFeedback struct {
+		SafetyRatings []struct {
+			Category    string `json:"category"`
+			Probability string `json:"probability"`
+		} `json:"safetyRatings"`
+	} `json:"promptFeedback"`
+	Error struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+		Status  string `json:"status"`
+		Details []struct {
+			Type            string `json:"@type"`
+			FieldViolations []struct {
+				Field       string `json:"field"`
+				Description string `json:"description"`
+			} `json:"fieldViolations"`
+		} `json:"details"`
+	} `json:"error"`
+}
+
+func ChatProxy(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
+	usagelog := store.Tokens{Model: chatReq.Model}
+
+	token, _ := c.Get("localuser")
+
+	lu, err := store.GetUserByToken(token.(string))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+	usagelog.UserID = int(lu.ID)
+	var prompts []genai.Part
+	var prompt string
+	for _, msg := range chatReq.Messages {
+		switch ct := msg.Content.(type) {
+		case string:
+			prompt += "<" + msg.Role + ">: " + msg.Content.(string) + "\n"
+			prompts = append(prompts, genai.Text("<"+msg.Role+">: "+msg.Content.(string)))
+		case []any:
+			for _, item := range ct {
+				if m, ok := item.(map[string]interface{}); ok {
+					if m["type"] == "text" {
+						prompt += "<" + msg.Role + ">: " + m["text"].(string) + "\n"
+						prompts = append(prompts, genai.Text("<"+msg.Role+">: "+m["text"].(string)))
+					} else if m["type"] == "image_url" {
+						if url, ok := m["image_url"].(map[string]interface{}); ok {
+							if strings.HasPrefix(url["url"].(string), "http") {
+								fmt.Println("网络图片:", url["url"].(string))
+							} else if strings.HasPrefix(url["url"].(string), "data:image") {
+								fmt.Println("base64:", url["url"].(string)[:20])
+								var mime string
+								// openai 会以 data:image 开头，则去掉 data:image/png;base64, 和 data:image/jpeg;base64,
+								if strings.HasPrefix(url["url"].(string), "data:image/png") {
+									mime = "image/png"
+								} else if strings.HasPrefix(url["url"].(string), "data:image/jpeg") {
+									mime = "image/jpeg"
+								} else {
+									c.JSON(http.StatusInternalServerError, gin.H{"error": "Unsupported image format"})
+									return
+								}
+								imageString := strings.Split(url["url"].(string), ",")[1]
+								imageBytes, err := base64.StdEncoding.DecodeString(imageString)
+								if err != nil {
+									c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+									return
+								}
+								prompts = append(prompts, genai.Blob{MIMEType: mime, Data: imageBytes})
+							}
+						}
+					}
+				}
+			}
+		default:
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error": gin.H{
+					"message": "Invalid content type",
+				},
+			})
+			return
+		}
+		if len(chatReq.Tools) > 0 {
+			tooljson, _ := json.Marshal(chatReq.Tools)
+			prompt += "<tools>: " + string(tooljson) + "\n"
+		}
+	}
+
+	usagelog.PromptCount = tokenizer.NumTokensFromStr(prompt, chatReq.Model)
+
+	onekey, err := store.SelectKeyCache("google")
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	ctx := context.Background()
+
+	client, err := genai.NewClient(ctx, option.WithAPIKey(onekey.Key))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	defer client.Close()
+
+	model := client.GenerativeModel(chatReq.Model)
+	model.Tools = []*genai.Tool{}
+
+	iter := model.GenerateContentStream(ctx, prompts...)
+	datachan := make(chan string)
+	// closechan := make(chan error)
+	var result string
+	go func() {
+		for {
+			resp, err := iter.Next()
+			if err == iterator.Done {
+
+				var chatResp openai.ChatCompletionStreamResponse
+				chatResp.Model = chatReq.Model
+				choice := openai.Choice{}
+				choice.FinishReason = "stop"
+				chatResp.Choices = append(chatResp.Choices, choice)
+				datachan <- "data: " + string(chatResp.ByteJson())
+				close(datachan)
+				break
+			}
+			if err != nil {
+				log.Println(err)
+				var errResp openai.ErrResponse
+				errResp.Error.Code = "500"
+				errResp.Error.Message = err.Error()
+				datachan <- string(errResp.ByteJson())
+				close(datachan)
+				break
+			}
+			var content string
+			if resp.Candidates != nil && len(resp.Candidates) > 0 && len(resp.Candidates[0].Content.Parts) > 0 {
+				if s, ok := resp.Candidates[0].Content.Parts[0].(genai.Text); ok {
+					content = string(s)
+					result += content
+				}
+			} else {
+				continue
+			}
+
+			var chatResp openai.ChatCompletionStreamResponse
+			chatResp.Model = chatReq.Model
+			choice := openai.Choice{}
+			choice.Delta.Role = resp.Candidates[0].Content.Role
+			choice.Delta.Content = content
+			chatResp.Choices = append(chatResp.Choices, choice)
+
+			chunk := "data: " + string(chatResp.ByteJson()) + "\n\n"
+			datachan <- chunk
+		}
+	}()
+
+	c.Writer.Header().Set("Content-Type", "text/event-stream")
+	c.Writer.Header().Set("Cache-Control", "no-cache")
+	c.Writer.Header().Set("Connection", "keep-alive")
+	c.Writer.Header().Set("Transfer-Encoding", "chunked")
+	c.Writer.Header().Set("X-Accel-Buffering", "no")
+
+	c.Stream(func(w io.Writer) bool {
+		if data, ok := <-datachan; ok {
+			if strings.HasPrefix(data, "data: ") {
+				c.Writer.WriteString(data)
+				// c.Writer.WriteString("\n\n")
+			} else {
+				c.Writer.WriteHeader(http.StatusBadGateway)
+				c.Writer.WriteString(data)
+			}
+			c.Writer.Flush()
+			return true
+		}
+		go func() {
+
+		}()
+		return false
+	})
+}
--- a/llm/google/v2/chat.go
+++ b/llm/google/v2/chat.go
@@ -0,0 +1,228 @@
+// https://github.com/google-gemini/api-examples/
+
+package google
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"opencatd-open/internal/model"
+	"opencatd-open/llm"
+	"os"
+	"strings"
+
+	"github.com/sashabaranov/go-openai"
+
+	"google.golang.org/genai"
+)
+
+type Gemini struct {
+	Ctx        context.Context
+	Client     *genai.Client
+	ApiKey     *model.ApiKey
+	tokenUsage *llm.TokenUsage
+
+	Done chan struct{}
+}
+
+func NewGemini(ctx context.Context, apiKey *model.ApiKey) (*Gemini, error) {
+	hc := http.DefaultClient
+	if os.Getenv("LOCAL_PROXY") != "" {
+		proxyUrl, err := url.Parse(os.Getenv("LOCAL_PROXY"))
+		if err == nil {
+			hc = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
+		}
+	}
+	client, err := genai.NewClient(ctx, &genai.ClientConfig{
+		APIKey:     *apiKey.ApiKey,
+		Backend:    genai.BackendGeminiAPI,
+		HTTPClient: hc,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Gemini{
+		Ctx:        context.Background(),
+		Client:     client,
+		ApiKey:     apiKey,
+		tokenUsage: &llm.TokenUsage{},
+		Done:       make(chan struct{}),
+	}, nil
+
+}
+
+func (g *Gemini) Chat(ctx context.Context, chatReq llm.ChatRequest) (*llm.ChatResponse, error) {
+	var content []*genai.Content
+	if len(chatReq.Messages) > 0 {
+		for _, msg := range chatReq.Messages {
+			var role genai.Role
+			if msg.Role == "user" || msg.Role == "system" {
+				role = genai.RoleUser
+			} else {
+				role = genai.RoleModel
+			}
+
+			if len(msg.MultiContent) > 0 {
+				for _, c := range msg.MultiContent {
+					var parts []*genai.Part
+
+					if c.Type == "text" {
+						parts = append(parts, genai.NewPartFromText(c.Text))
+					}
+					if c.Type == "image_url" {
+						if strings.HasPrefix(c.ImageURL.URL, "http") {
+							continue
+						}
+						if strings.HasPrefix(c.ImageURL.URL, "data:image") {
+							var mediaType string
+							if strings.HasPrefix(c.ImageURL.URL, "data:image/jpeg") {
+								mediaType = "image/jpeg"
+							}
+							if strings.HasPrefix(c.ImageURL.URL, "data:image/png") {
+								mediaType = "image/png"
+							}
+							imageString := strings.Split(c.ImageURL.URL, ",")[1]
+							imageBytes, _ := base64.StdEncoding.DecodeString(imageString)
+
+							parts = append(parts, genai.NewPartFromBytes(imageBytes, mediaType))
+						}
+
+					}
+					content = append(content, genai.NewContentFromParts(parts, role))
+				}
+			} else {
+				content = append(content, genai.NewContentFromText(msg.Content, role))
+			}
+
+		}
+	}
+
+	tools := []*genai.Tool{{GoogleSearch: &genai.GoogleSearch{}}}
+	response, err := g.Client.Models.GenerateContent(g.Ctx,
+		chatReq.Model,
+		content,
+		&genai.GenerateContentConfig{Tools: tools})
+	if err != nil {
+		return nil, err
+	}
+
+	if response.UsageMetadata != nil {
+		g.tokenUsage.PromptTokens += int(response.UsageMetadata.PromptTokenCount)
+		g.tokenUsage.CompletionTokens += int(response.UsageMetadata.CandidatesTokenCount)
+		g.tokenUsage.ToolsTokens += int(response.UsageMetadata.ToolUsePromptTokenCount)
+		g.tokenUsage.TotalTokens += int(response.UsageMetadata.TotalTokenCount)
+	}
+
+	// var text string
+	// if response.Candidates != nil && response.Candidates[0].Content != nil {
+	// 	for _, part := range response.Candidates[0].Content.Parts {
+	// 		text += part.Text
+	// 	}
+	// }
+
+	return &llm.ChatResponse{
+		Model: response.ModelVersion,
+		Choices: []openai.ChatCompletionChoice{
+			{
+				Message:      openai.ChatCompletionMessage{Content: response.Text(), Role: "assistant"},
+				FinishReason: openai.FinishReason(response.Candidates[0].FinishReason),
+			},
+		},
+		Usage: openai.Usage{PromptTokens: g.tokenUsage.PromptTokens + g.tokenUsage.ToolsTokens, CompletionTokens: g.tokenUsage.CompletionTokens, TotalTokens: g.tokenUsage.TotalTokens},
+	}, nil
+
+}
+
+func (g *Gemini) StreamChat(ctx context.Context, chatReq llm.ChatRequest) (chan *llm.StreamChatResponse, error) {
+	var contents []*genai.Content
+	if len(chatReq.Messages) > 0 {
+		for _, msg := range chatReq.Messages {
+			var role genai.Role
+			if msg.Role == "user" {
+				role = genai.RoleUser
+			} else {
+				role = genai.RoleModel
+			}
+
+			if len(msg.MultiContent) > 0 {
+				for _, c := range msg.MultiContent {
+					var parts []*genai.Part
+
+					if c.Type == "text" {
+						parts = append(parts, genai.NewPartFromText(c.Text))
+					}
+					if c.Type == "image_url" {
+						if strings.HasPrefix(c.ImageURL.URL, "http") {
+							continue
+						}
+						if strings.HasPrefix(c.ImageURL.URL, "data:image") {
+							var mediaType string
+							if strings.HasPrefix(c.ImageURL.URL, "data:image/jpeg") {
+								mediaType = "image/jpeg"
+							}
+							if strings.HasPrefix(c.ImageURL.URL, "data:image/png") {
+								mediaType = "image/png"
+							}
+							imageString := strings.Split(c.ImageURL.URL, ",")[1]
+							imageBytes, _ := base64.StdEncoding.DecodeString(imageString)
+
+							parts = append(parts, genai.NewPartFromBytes(imageBytes, mediaType))
+						}
+
+					}
+					contents = append(contents, genai.NewContentFromParts(parts, role))
+				}
+			} else {
+				contents = append(contents, genai.NewContentFromText(msg.Content, role))
+			}
+
+		}
+	}
+
+	datachan := make(chan *llm.StreamChatResponse)
+	var generr error
+
+	tools := []*genai.Tool{{GoogleSearch: &genai.GoogleSearch{}}}
+
+	go func() {
+		defer close(datachan)
+		for result, err := range g.Client.Models.GenerateContentStream(g.Ctx, chatReq.Model, contents, &genai.GenerateContentConfig{Tools: tools}) {
+			if err != nil {
+				fmt.Println(err)
+				generr = err
+				return
+			}
+			if result.UsageMetadata != nil {
+				g.tokenUsage.PromptTokens += int(result.UsageMetadata.PromptTokenCount)
+				g.tokenUsage.CompletionTokens += int(result.UsageMetadata.CandidatesTokenCount)
+				g.tokenUsage.ToolsTokens += int(result.UsageMetadata.ToolUsePromptTokenCount)
+				g.tokenUsage.TotalTokens += int(result.UsageMetadata.TotalTokenCount)
+			}
+
+			datachan <- &llm.StreamChatResponse{
+				Model: result.ModelVersion,
+				Choices: []openai.ChatCompletionStreamChoice{
+					{
+						Delta: openai.ChatCompletionStreamChoiceDelta{
+							Role: "assistant",
+							// Content: result.Candidates[0].Content.Parts[0].Text,
+							Content: result.Text(),
+						},
+						FinishReason: openai.FinishReason(result.Candidates[0].FinishReason),
+					},
+				},
+				Usage: &openai.Usage{PromptTokens: g.tokenUsage.PromptTokens + g.tokenUsage.ToolsTokens, CompletionTokens: g.tokenUsage.CompletionTokens, TotalTokens: g.tokenUsage.TotalTokens},
+			}
+
+		}
+	}()
+
+	return datachan, generr
+}
+
+func (g *Gemini) GetTokenUsage() *llm.TokenUsage {
+	return g.tokenUsage
+}
--- a/llm/llm.go
+++ b/llm/llm.go
@@ -0,0 +1,20 @@
+package llm
+
+import (
+	"context"
+	"opencatd-open/internal/model"
+)
+
+type LLM interface {
+	Chat(ctx context.Context, req ChatRequest) (*ChatResponse, error)
+	StreamChat(ctx context.Context, req ChatRequest) (chan *StreamChatResponse, error)
+	GetTokenUsage() *TokenUsage
+}
+
+type llm struct {
+	ApiKey   *model.ApiKey
+	Usage    *model.Usage
+	tools    any   // TODO
+	Messages []any // TODO
+	llm      LLM
+}
--- a/llm/openai/chat.go
+++ b/llm/openai/chat.go
@@ -0,0 +1,178 @@
+package openai
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+)
+
+const (
+	// https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation#latest-preview-api-releases
+	AzureApiVersion    = "2024-10-21"
+	BaseHost           = "api.openai.com"
+	OpenAI_Endpoint    = "https://api.openai.com/v1/chat/completions"
+	Github_Marketplace = "https://models.inference.ai.azure.com/chat/completions"
+)
+
+var (
+	Custom_Endpoint    string
+	AIGateWay_Endpoint string // "https://gateway.ai.cloudflare.com/v1/431ba10f11200d544922fbca177aaa7f/openai/openai/chat/completions"
+)
+
+func init() {
+	if os.Getenv("OpenAI_Endpoint") != "" {
+		Custom_Endpoint = os.Getenv("OpenAI_Endpoint")
+	}
+	if os.Getenv("AIGateWay_Endpoint") != "" {
+		AIGateWay_Endpoint = os.Getenv("AIGateWay_Endpoint")
+	}
+}
+
+// Vision Content
+type VisionContent struct {
+	Type     string          `json:"type,omitempty"`
+	Text     string          `json:"text,omitempty"`
+	ImageURL *VisionImageURL `json:"image_url,omitempty"`
+}
+type VisionImageURL struct {
+	URL    string `json:"url,omitempty"`
+	Detail string `json:"detail,omitempty"`
+}
+
+type ChatCompletionMessage struct {
+	Role    string `json:"role"`
+	Content any    `json:"content"`
+	Name    string `json:"name,omitempty"`
+	// MultiContent []VisionContent
+}
+
+type FunctionDefinition struct {
+	Name        string `json:"name"`
+	Description string `json:"description,omitempty"`
+	Parameters  any    `json:"parameters"`
+}
+
+type Tool struct {
+	Type     string              `json:"type"`
+	Function *FunctionDefinition `json:"function,omitempty"`
+}
+
+type StreamOption struct {
+	IncludeUsage bool `json:"include_usage,omitempty"`
+}
+
+type ChatCompletionRequest struct {
+	Model            string                  `json:"model"`
+	Messages         []ChatCompletionMessage `json:"messages"`
+	MaxTokens        int                     `json:"max_tokens,omitempty"`
+	Temperature      float64                 `json:"temperature,omitempty"`
+	TopP             float64                 `json:"top_p,omitempty"`
+	N                int                     `json:"n,omitempty"`
+	Stream           bool                    `json:"stream"`
+	Stop             []string                `json:"stop,omitempty"`
+	PresencePenalty  float64                 `json:"presence_penalty,omitempty"`
+	FrequencyPenalty float64                 `json:"frequency_penalty,omitempty"`
+	LogitBias        map[string]int          `json:"logit_bias,omitempty"`
+	User             string                  `json:"user,omitempty"`
+	// Functions        []FunctionDefinition       `json:"functions,omitempty"`
+	// FunctionCall     any                        `json:"function_call,omitempty"`
+	Tools             []Tool `json:"tools,omitempty"`
+	ParallelToolCalls bool   `json:"parallel_tool_calls,omitempty"`
+	// ToolChoice any    `json:"tool_choice,omitempty"`
+	StreamOptions *StreamOption `json:"stream_options,omitempty"`
+}
+
+func (c ChatCompletionRequest) ToByteJson() []byte {
+	bytejson, _ := json.Marshal(c)
+	return bytejson
+}
+
+type ToolCall struct {
+	ID       string `json:"id"`
+	Type     string `json:"type"`
+	Function struct {
+		Name      string `json:"name"`
+		Arguments string `json:"arguments"`
+	} `json:"function"`
+}
+
+type ChatCompletionResponse struct {
+	ID      string `json:"id,omitempty"`
+	Object  string `json:"object,omitempty"`
+	Created int    `json:"created,omitempty"`
+	Model   string `json:"model,omitempty"`
+	Choices []struct {
+		Index   int `json:"index,omitempty"`
+		Message struct {
+			Role      string     `json:"role,omitempty"`
+			Content   string     `json:"content,omitempty"`
+			ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+		} `json:"message,omitempty"`
+		Logprobs     string `json:"logprobs,omitempty"`
+		FinishReason string `json:"finish_reason,omitempty"`
+	} `json:"choices,omitempty"`
+	Usage struct {
+		PromptTokens        int `json:"prompt_tokens,omitempty"`
+		CompletionTokens    int `json:"completion_tokens,omitempty"`
+		TotalTokens         int `json:"total_tokens,omitempty"`
+		PromptTokensDetails struct {
+			CachedTokens int `json:"cached_tokens,omitempty"`
+			AudioTokens  int `json:"audio_tokens,omitempty"`
+		} `json:"prompt_tokens_details,omitempty"`
+		CompletionTokensDetails struct {
+			ReasoningTokens          int `json:"reasoning_tokens,omitempty"`
+			AudioTokens              int `json:"audio_tokens,omitempty"`
+			AcceptedPredictionTokens int `json:"accepted_prediction_tokens,omitempty"`
+			RejectedPredictionTokens int `json:"rejected_prediction_tokens,omitempty"`
+		} `json:"completion_tokens_details,omitempty"`
+	} `json:"usage,omitempty"`
+	SystemFingerprint string `json:"system_fingerprint,omitempty"`
+}
+
+type Choice struct {
+	Index int `json:"index"`
+	Delta struct {
+		Role      string     `json:"role"`
+		Content   string     `json:"content"`
+		ToolCalls []ToolCall `json:"tool_calls"`
+	} `json:"delta"`
+	FinishReason string `json:"finish_reason"`
+	Usage        struct {
+		PromptTokens     int `json:"prompt_tokens"`
+		CompletionTokens int `json:"completion_tokens"`
+		TotalTokens      int `json:"total_tokens"`
+	} `json:"usage"`
+}
+
+type ChatCompletionStreamResponse struct {
+	ID      string   `json:"id"`
+	Object  string   `json:"object"`
+	Created int      `json:"created"`
+	Model   string   `json:"model"`
+	Choices []Choice `json:"choices"`
+}
+
+func (c *ChatCompletionStreamResponse) ByteJson() []byte {
+	bytejson, _ := json.Marshal(c)
+	return bytejson
+}
+
+func modelmap(in string) string {
+	// gpt-3.5-turbo -> gpt-35-turbo
+	if strings.Contains(in, ".") {
+		return strings.ReplaceAll(in, ".", "")
+	}
+	return in
+}
+
+type ErrResponse struct {
+	Error struct {
+		Message string `json:"message"`
+		Code    string `json:"code"`
+	} `json:"error"`
+}
+
+func (e *ErrResponse) ByteJson() []byte {
+	bytejson, _ := json.Marshal(e)
+	return bytejson
+}
--- a/llm/openai/dall-e.go
+++ b/llm/openai/dall-e.go
@@ -0,0 +1,149 @@
+package openai
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+	"strconv"
+
+	"github.com/duke-git/lancet/v2/slice"
+	"github.com/gin-gonic/gin"
+)
+
+const (
+	DalleEndpoint          = "https://api.openai.com/v1/images/generations"
+	DalleEditEndpoint      = "https://api.openai.com/v1/images/edits"
+	DalleVariationEndpoint = "https://api.openai.com/v1/images/variations"
+)
+
+type DallERequest struct {
+	Model          string `json:"model"`
+	Prompt         string `json:"prompt"`
+	N              int    `form:"n" json:"n,omitempty"`
+	Size           string `form:"size" json:"size,omitempty"`
+	Quality        string `json:"quality,omitempty"`         // standard,hd
+	Style          string `json:"style,omitempty"`           // vivid,natural
+	ResponseFormat string `json:"response_format,omitempty"` // url or b64_json
+}
+
+func DallEProxy(c *gin.Context) {
+
+	var dalleRequest DallERequest
+	if err := c.ShouldBind(&dalleRequest); err != nil {
+		c.JSON(400, gin.H{"error": err.Error()})
+		return
+	}
+	if dalleRequest.N == 0 {
+		dalleRequest.N = 1
+	}
+
+	if dalleRequest.Size == "" {
+		dalleRequest.Size = "512x512"
+	}
+
+	model := dalleRequest.Model
+
+	var chatlog store.Tokens
+	chatlog.CompletionCount = dalleRequest.N
+
+	if model == "dall-e" {
+		model = "dall-e-2"
+	}
+	model = model + "." + dalleRequest.Size
+
+	if dalleRequest.Model == "dall-e-2" || dalleRequest.Model == "dall-e" {
+		if !slice.Contain([]string{"256x256", "512x512", "1024x1024"}, dalleRequest.Size) {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"error": gin.H{
+					"message": fmt.Sprintf("Invalid size: %s for %s", dalleRequest.Size, dalleRequest.Model),
+				},
+			})
+			return
+		}
+	} else if dalleRequest.Model == "dall-e-3" {
+		if !slice.Contain([]string{"256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"}, dalleRequest.Size) {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"error": gin.H{
+					"message": fmt.Sprintf("Invalid size: %s for %s", dalleRequest.Size, dalleRequest.Model),
+				},
+			})
+			return
+		}
+		if dalleRequest.Quality == "hd" {
+			model = model + ".hd"
+		}
+	} else {
+		c.JSON(http.StatusBadRequest, gin.H{
+			"error": gin.H{
+				"message": fmt.Sprintf("Invalid model: %s", dalleRequest.Model),
+			},
+		})
+		return
+	}
+	chatlog.Model = model
+
+	token, _ := c.Get("localuser")
+
+	lu, err := store.GetUserByToken(token.(string))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+	chatlog.UserID = int(lu.ID)
+
+	key, err := store.SelectKeyCache("openai")
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+
+	targetURL, _ := url.Parse(DalleEndpoint)
+	proxy := httputil.NewSingleHostReverseProxy(targetURL)
+	proxy.Director = func(req *http.Request) {
+		req.Header.Set("Authorization", "Bearer "+key.Key)
+		req.Header.Set("Content-Type", "application/json")
+
+		req.Host = targetURL.Host
+		req.URL.Scheme = targetURL.Scheme
+		req.URL.Host = targetURL.Host
+		req.URL.Path = targetURL.Path
+		req.URL.RawPath = targetURL.RawPath
+		req.URL.RawQuery = targetURL.RawQuery
+
+		bytebody, _ := json.Marshal(dalleRequest)
+		req.Body = io.NopCloser(bytes.NewBuffer(bytebody))
+		req.ContentLength = int64(len(bytebody))
+		req.Header.Set("Content-Length", strconv.Itoa(len(bytebody)))
+	}
+
+	proxy.ModifyResponse = func(resp *http.Response) error {
+		if resp.StatusCode == http.StatusOK {
+			chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
+			chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
+			if err := store.Record(&chatlog); err != nil {
+				log.Println(err)
+			}
+			if err := store.SumDaily(chatlog.UserID); err != nil {
+				log.Println(err)
+			}
+		}
+		return nil
+	}
+
+	proxy.ServeHTTP(c.Writer, c.Request)
+}
--- a/llm/openai/handle_proxy.go
+++ b/llm/openai/handle_proxy.go
@@ -0,0 +1,215 @@
+package openai
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+func ChatProxy(c *gin.Context, chatReq *ChatCompletionRequest) {
+	usagelog := store.Tokens{Model: chatReq.Model}
+
+	token, _ := c.Get("localuser")
+
+	lu, err := store.GetUserByToken(token.(string))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+	usagelog.UserID = int(lu.ID)
+
+	var prompt string
+	for _, msg := range chatReq.Messages {
+		switch ct := msg.Content.(type) {
+		case string:
+			prompt += "<" + msg.Role + ">: " + msg.Content.(string) + "\n"
+		case []any:
+			for _, item := range ct {
+				if m, ok := item.(map[string]interface{}); ok {
+					if m["type"] == "text" {
+						prompt += "<" + msg.Role + ">: " + m["text"].(string) + "\n"
+					} else if m["type"] == "image_url" {
+						if url, ok := m["image_url"].(map[string]interface{}); ok {
+							fmt.Printf("  URL: %v\n", url["url"])
+							if strings.HasPrefix(url["url"].(string), "http") {
+								fmt.Println("网络图片:", url["url"].(string))
+							}
+						}
+					}
+				}
+			}
+		default:
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error": gin.H{
+					"message": "Invalid content type",
+				},
+			})
+			return
+		}
+		if len(chatReq.Tools) > 0 {
+			tooljson, _ := json.Marshal(chatReq.Tools)
+			prompt += "<tools>: " + string(tooljson) + "\n"
+		}
+	}
+	switch chatReq.Model {
+	case "gpt-4o", "gpt-4o-mini", "chatgpt-4o-latest":
+		chatReq.MaxTokens = 16384
+	}
+	if chatReq.Stream {
+		chatReq.StreamOptions = &StreamOption{IncludeUsage: true}
+	}
+
+	usagelog.PromptCount = tokenizer.NumTokensFromStr(prompt, chatReq.Model)
+
+	// onekey, err := store.SelectKeyCache("openai")
+	onekey, err := store.SelectKeyCacheByModel(chatReq.Model)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	var req *http.Request
+
+	switch onekey.ApiType {
+	case "github":
+		req, err = http.NewRequest(c.Request.Method, Github_Marketplace, bytes.NewReader(chatReq.ToByteJson()))
+		req.Header = c.Request.Header
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", onekey.Key))
+	case "azure":
+		var buildurl string
+		if onekey.EndPoint != "" {
+			buildurl = fmt.Sprintf("%s/openai/deployments/%s/chat/completions?api-version=%s", onekey.EndPoint, modelmap(chatReq.Model), AzureApiVersion)
+		} else {
+			buildurl = fmt.Sprintf("https://%s.openai.azure.com/openai/deployments/%s/chat/completions?api-version=%s", onekey.ResourceNmae, modelmap(chatReq.Model), AzureApiVersion)
+		}
+		req, err = http.NewRequest(c.Request.Method, buildurl, bytes.NewReader(chatReq.ToByteJson()))
+		req.Header = c.Request.Header
+		req.Header.Set("api-key", onekey.Key)
+	default:
+		req, err = http.NewRequest(c.Request.Method, OpenAI_Endpoint, bytes.NewReader(chatReq.ToByteJson())) // default endpoint
+
+		if AIGateWay_Endpoint != "" { // cloudflare gateway的endpoint
+			req, err = http.NewRequest(c.Request.Method, AIGateWay_Endpoint, bytes.NewReader(chatReq.ToByteJson()))
+		}
+		if Custom_Endpoint != "" { // 自定义endpoint
+			req, err = http.NewRequest(c.Request.Method, Custom_Endpoint, bytes.NewReader(chatReq.ToByteJson()))
+		}
+		if onekey.EndPoint != "" { // 优先key的endpoint
+			req, err = http.NewRequest(c.Request.Method, onekey.EndPoint+c.Request.RequestURI, bytes.NewReader(chatReq.ToByteJson()))
+		}
+
+		req.Header = c.Request.Header
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", onekey.Key))
+	}
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	defer resp.Body.Close()
+
+	var result string
+	if chatReq.Stream {
+		for key, value := range resp.Header {
+			for _, v := range value {
+				c.Writer.Header().Add(key, v)
+			}
+		}
+		c.Writer.WriteHeader(resp.StatusCode)
+		teeReader := io.TeeReader(resp.Body, c.Writer)
+		// 流式响应
+		scanner := bufio.NewScanner(teeReader)
+
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if len(line) > 0 && bytes.HasPrefix(line, []byte("data: ")) {
+				if bytes.HasPrefix(line, []byte("data: [DONE]")) {
+					break
+				}
+				var opiResp ChatCompletionStreamResponse
+				line = bytes.Replace(line, []byte("data: "), []byte(""), -1)
+				line = bytes.TrimSpace(line)
+				if err := json.Unmarshal(line, &opiResp); err != nil {
+					continue
+				}
+
+				if opiResp.Choices != nil && len(opiResp.Choices) > 0 {
+					if opiResp.Choices[0].Delta.Role != "" {
+						result += "<" + opiResp.Choices[0].Delta.Role + "> "
+					}
+					result += opiResp.Choices[0].Delta.Content // 计算Content Token
+
+					if len(opiResp.Choices[0].Delta.ToolCalls) > 0 { // 计算ToolCalls token
+						if opiResp.Choices[0].Delta.ToolCalls[0].Function.Name != "" {
+							result += "name:" + opiResp.Choices[0].Delta.ToolCalls[0].Function.Name + " arguments:"
+						}
+						result += opiResp.Choices[0].Delta.ToolCalls[0].Function.Arguments
+					}
+				} else {
+					continue
+				}
+			}
+
+		}
+	} else {
+
+		// 处理非流式响应
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			fmt.Println("Error reading response body:", err)
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		var opiResp ChatCompletionResponse
+		if err := json.Unmarshal(body, &opiResp); err != nil {
+			log.Println("Error parsing JSON:", err)
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		if opiResp.Choices != nil && len(opiResp.Choices) > 0 {
+			if opiResp.Choices[0].Message.Role != "" {
+				result += "<" + opiResp.Choices[0].Message.Role + "> "
+			}
+			result += opiResp.Choices[0].Message.Content
+
+			if len(opiResp.Choices[0].Message.ToolCalls) > 0 {
+				if opiResp.Choices[0].Message.ToolCalls[0].Function.Name != "" {
+					result += "name:" + opiResp.Choices[0].Message.ToolCalls[0].Function.Name + " arguments:"
+				}
+				result += opiResp.Choices[0].Message.ToolCalls[0].Function.Arguments
+			}
+
+		}
+		for k, v := range resp.Header {
+			c.Writer.Header().Set(k, v[0])
+		}
+
+		c.JSON(http.StatusOK, opiResp)
+	}
+	usagelog.CompletionCount = tokenizer.NumTokensFromStr(result, chatReq.Model)
+	usagelog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(usagelog.Model, usagelog.PromptCount, usagelog.CompletionCount))
+	if err := store.Record(&usagelog); err != nil {
+		log.Println(err)
+	}
+	if err := store.SumDaily(usagelog.UserID); err != nil {
+		log.Println(err)
+	}
+}
--- a/llm/openai/realtime.go
+++ b/llm/openai/realtime.go
@@ -0,0 +1,197 @@
+/*
+https://platform.openai.com/docs/guides/realtime
+https://learn.microsoft.com/zh-cn/azure/ai-services/openai/how-to/audio-real-time
+
+wss://my-eastus2-openai-resource.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-1001
+*/
+package openai
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+	"os"
+
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"golang.org/x/sync/errgroup"
+)
+
+// "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01"
+const realtimeURL = "wss://api.openai.com/v1/realtime"
+const azureRealtimeURL = "wss://%s.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"
+
+var upgrader = websocket.Upgrader{
+	CheckOrigin: func(r *http.Request) bool {
+		return true
+	},
+}
+
+type Message struct {
+	Type     string   `json:"type"`
+	Response Response `json:"response"`
+}
+
+type Response struct {
+	Modalities   []string `json:"modalities"`
+	Instructions string   `json:"instructions"`
+}
+
+type RealTimeResponse struct {
+	Type     string `json:"type"`
+	EventID  string `json:"event_id"`
+	Response struct {
+		Object        string `json:"object"`
+		ID            string `json:"id"`
+		Status        string `json:"status"`
+		StatusDetails any    `json:"status_details"`
+		Output        []struct {
+			ID      string `json:"id"`
+			Object  string `json:"object"`
+			Type    string `json:"type"`
+			Status  string `json:"status"`
+			Role    string `json:"role"`
+			Content []struct {
+				Type       string `json:"type"`
+				Transcript string `json:"transcript"`
+			} `json:"content"`
+		} `json:"output"`
+		Usage Usage `json:"usage"`
+	} `json:"response"`
+}
+
+type Usage struct {
+	TotalTokens       int `json:"total_tokens"`
+	InputTokens       int `json:"input_tokens"`
+	OutputTokens      int `json:"output_tokens"`
+	InputTokenDetails struct {
+		CachedTokens int `json:"cached_tokens"`
+		TextTokens   int `json:"text_tokens"`
+		AudioTokens  int `json:"audio_tokens"`
+	} `json:"input_token_details"`
+	OutputTokenDetails struct {
+		TextTokens  int `json:"text_tokens"`
+		AudioTokens int `json:"audio_tokens"`
+	} `json:"output_token_details"`
+}
+
+func RealTimeProxy(c *gin.Context) {
+	log.Println(c.Request.URL.String())
+	var model string = c.Query("model")
+	value := url.Values{}
+	value.Add("model", model)
+	realtimeURL := realtimeURL + "?" + value.Encode()
+
+	// 升级 HTTP 连接为 WebSocket
+	clientConn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
+	if err != nil {
+		log.Println("Upgrade error:", err)
+		return
+	}
+	defer clientConn.Close()
+
+	apikey, err := store.SelectKeyCacheByModel(model)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	// 连接到 OpenAI WebSocket
+	headers := http.Header{"OpenAI-Beta": []string{"realtime=v1"}}
+
+	if apikey.ApiType == "azure" {
+		headers.Set("api-key", apikey.Key)
+		if apikey.EndPoint != "" {
+			realtimeURL = fmt.Sprintf("%s/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview", apikey.EndPoint)
+		} else {
+			realtimeURL = fmt.Sprintf(azureRealtimeURL, apikey.ResourceNmae)
+		}
+	} else {
+		headers.Set("Authorization", "Bearer "+apikey.Key)
+	}
+
+	conn := websocket.DefaultDialer
+	if os.Getenv("LOCAL_PROXY") != "" {
+		proxyUrl, _ := url.Parse(os.Getenv("LOCAL_PROXY"))
+		conn.Proxy = http.ProxyURL(proxyUrl)
+	}
+
+	openAIConn, _, err := conn.Dial(realtimeURL, headers)
+	if err != nil {
+		log.Println("OpenAI dial error:", err)
+		return
+	}
+	defer openAIConn.Close()
+
+	ctx, cancel := context.WithCancel(c.Request.Context())
+	defer cancel()
+
+	g, ctx := errgroup.WithContext(ctx)
+
+	g.Go(func() error {
+		return forwardMessages(ctx, c, clientConn, openAIConn)
+	})
+
+	g.Go(func() error {
+		return forwardMessages(ctx, c, openAIConn, clientConn)
+	})
+
+	if err := g.Wait(); err != nil {
+		log.Println("Error in message forwarding:", err)
+		return
+	}
+
+}
+
+func forwardMessages(ctx context.Context, c *gin.Context, src, dst *websocket.Conn) error {
+	usagelog := store.Tokens{Model: "gpt-4o-realtime-preview"}
+
+	token, _ := c.Get("localuser")
+
+	lu, err := store.GetUserByToken(token.(string))
+	if err != nil {
+		return err
+	}
+	usagelog.UserID = int(lu.ID)
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			messageType, message, err := src.ReadMessage()
+			if err != nil {
+				if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
+					return nil // 正常关闭，不报错
+				}
+				return err
+			}
+			if messageType == websocket.TextMessage {
+				var usage Usage
+				err := json.Unmarshal(message, &usage)
+				if err == nil {
+					usagelog.PromptCount += usage.InputTokens
+					usagelog.CompletionCount += usage.OutputTokens
+				}
+
+			}
+			err = dst.WriteMessage(messageType, message)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	defer func() {
+		usagelog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(usagelog.Model, usagelog.PromptCount, usagelog.CompletionCount))
+		if err := store.Record(&usagelog); err != nil {
+			log.Println(err)
+		}
+		if err := store.SumDaily(usagelog.UserID); err != nil {
+			log.Println(err)
+		}
+	}()
+	return nil
+}
--- a/llm/openai/tts.go
+++ b/llm/openai/tts.go
@@ -0,0 +1,93 @@
+package openai
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+
+	"github.com/gin-gonic/gin"
+)
+
+const (
+	SpeechEndpoint = "https://api.openai.com/v1/audio/speech"
+)
+
+type SpeechRequest struct {
+	Model string `json:"model"`
+	Input string `json:"input"`
+	Voice string `json:"voice"`
+}
+
+func SpeechProxy(c *gin.Context) {
+	var chatreq SpeechRequest
+	if err := c.ShouldBindJSON(&chatreq); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	var chatlog store.Tokens
+	chatlog.Model = chatreq.Model
+	chatlog.CompletionCount = len(chatreq.Input)
+
+	token, _ := c.Get("localuser")
+
+	lu, err := store.GetUserByToken(token.(string))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+	chatlog.UserID = int(lu.ID)
+
+	key, err := store.SelectKeyCache("openai")
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+
+	targetURL, _ := url.Parse(SpeechEndpoint)
+	proxy := httputil.NewSingleHostReverseProxy(targetURL)
+
+	proxy.Director = func(req *http.Request) {
+		req.Header = c.Request.Header
+		req.Header["Authorization"] = []string{"Bearer " + key.Key}
+		req.Host = targetURL.Host
+		req.URL.Scheme = targetURL.Scheme
+		req.URL.Host = targetURL.Host
+		req.URL.Path = targetURL.Path
+		req.URL.RawPath = targetURL.RawPath
+
+		reqBytes, _ := json.Marshal(chatreq)
+		req.Body = io.NopCloser(bytes.NewReader(reqBytes))
+		req.ContentLength = int64(len(reqBytes))
+
+	}
+	proxy.ModifyResponse = func(resp *http.Response) error {
+		if resp.StatusCode == http.StatusOK {
+			chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
+			chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
+			if err := store.Record(&chatlog); err != nil {
+				log.Println(err)
+			}
+			if err := store.SumDaily(chatlog.UserID); err != nil {
+				log.Println(err)
+			}
+		}
+		return nil
+	}
+	proxy.ServeHTTP(c.Writer, c.Request)
+}
--- a/llm/openai/whisper.go
+++ b/llm/openai/whisper.go
@@ -0,0 +1,177 @@
+package openai
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"mime/multipart"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"opencatd-open/pkg/tokenizer"
+	"opencatd-open/store"
+	"path/filepath"
+	"time"
+
+	"github.com/faiface/beep"
+	"github.com/faiface/beep/mp3"
+	"github.com/faiface/beep/wav"
+	"github.com/gin-gonic/gin"
+	"gopkg.in/vansante/go-ffprobe.v2"
+)
+
+func WhisperProxy(c *gin.Context) {
+	var chatlog store.Tokens
+
+	byteBody, _ := io.ReadAll(c.Request.Body)
+	c.Request.Body = io.NopCloser(bytes.NewBuffer(byteBody))
+
+	model, _ := c.GetPostForm("model")
+
+	key, err := store.SelectKeyCache("openai")
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+
+	chatlog.Model = model
+
+	token, _ := c.Get("localuser")
+
+	lu, err := store.GetUserByToken(token.(string))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+	chatlog.UserID = int(lu.ID)
+
+	if err := ParseWhisperRequestTokens(c, &chatlog, byteBody); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error": gin.H{
+				"message": err.Error(),
+			},
+		})
+		return
+	}
+	if key.EndPoint == "" {
+		key.EndPoint = "https://api.openai.com"
+	}
+	targetUrl, _ := url.ParseRequestURI(key.EndPoint + c.Request.URL.String())
+	log.Println(targetUrl)
+	proxy := httputil.NewSingleHostReverseProxy(targetUrl)
+	proxy.Director = func(req *http.Request) {
+		req.Host = targetUrl.Host
+		req.URL.Scheme = targetUrl.Scheme
+		req.URL.Host = targetUrl.Host
+
+		req.Header.Set("Authorization", "Bearer "+key.Key)
+	}
+
+	proxy.ModifyResponse = func(resp *http.Response) error {
+		if resp.StatusCode != http.StatusOK {
+			return nil
+		}
+		chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
+		chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
+		if err := store.Record(&chatlog); err != nil {
+			log.Println(err)
+		}
+		if err := store.SumDaily(chatlog.UserID); err != nil {
+			log.Println(err)
+		}
+		return nil
+	}
+	proxy.ServeHTTP(c.Writer, c.Request)
+}
+
+func probe(fileReader io.Reader) (time.Duration, error) {
+	ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancelFn()
+
+	data, err := ffprobe.ProbeReader(ctx, fileReader)
+	if err != nil {
+		return 0, err
+	}
+
+	duration := data.Format.DurationSeconds
+	pduration, err := time.ParseDuration(fmt.Sprintf("%fs", duration))
+	if err != nil {
+		return 0, fmt.Errorf("Error parsing duration: %s", err)
+	}
+	return pduration, nil
+}
+
+func getAudioDuration(file *multipart.FileHeader) (time.Duration, error) {
+	var (
+		streamer beep.StreamSeekCloser
+		format   beep.Format
+		err      error
+	)
+
+	f, err := file.Open()
+	defer f.Close()
+
+	// Get the file extension to determine the audio file type
+	fileType := filepath.Ext(file.Filename)
+
+	switch fileType {
+	case ".mp3":
+		streamer, format, err = mp3.Decode(f)
+	case ".wav":
+		streamer, format, err = wav.Decode(f)
+	case ".m4a":
+		duration, err := probe(f)
+		if err != nil {
+			return 0, err
+		}
+		return duration, nil
+	default:
+		return 0, errors.New("unsupported audio file format")
+	}
+
+	if err != nil {
+		return 0, err
+	}
+	defer streamer.Close()
+
+	// Calculate the audio file's duration.
+	numSamples := streamer.Len()
+	sampleRate := format.SampleRate
+	duration := time.Duration(numSamples) * time.Second / time.Duration(sampleRate)
+
+	return duration, nil
+}
+
+func ParseWhisperRequestTokens(c *gin.Context, usage *store.Tokens, byteBody []byte) error {
+	file, _ := c.FormFile("file")
+	model, _ := c.GetPostForm("model")
+	usage.Model = model
+
+	if file != nil {
+		duration, err := getAudioDuration(file)
+		if err != nil {
+			return fmt.Errorf("Error getting audio duration:%s", err)
+		}
+
+		if duration > 5*time.Minute {
+			return fmt.Errorf("Audio duration exceeds 5 minutes")
+		}
+		// 计算时长，四舍五入到最接近的秒数
+		usage.PromptCount = int(duration.Round(time.Second).Seconds())
+	}
+
+	c.Request.Body = io.NopCloser(bytes.NewBuffer(byteBody))
+
+	return nil
+}
--- a/llm/openai_compatible/chat.go
+++ b/llm/openai_compatible/chat.go
@@ -0,0 +1,221 @@
+package openai_compatible
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"opencatd-open/internal/model"
+	"opencatd-open/internal/utils"
+	"opencatd-open/llm"
+	"os"
+	"strings"
+)
+
+// https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation#latest-preview-api-releases
+const AzureApiVersion = "2024-10-21"
+const defaultOpenAICompatibleEndpoint = "https://api.openai.com/v1/chat/completions"
+const Github_Marketplace = "https://models.inference.ai.azure.com/chat/completions"
+
+type OpenAICompatible struct {
+	Client     *http.Client
+	ApiKey     *model.ApiKey
+	tokenUsage *llm.TokenUsage
+	Params     map[string]interface{}
+	Done       chan struct{}
+}
+
+func NewOpenAICompatible(apikey *model.ApiKey) (*OpenAICompatible, error) {
+	hc := http.DefaultClient
+	if os.Getenv("LOCAL_PROXY") != "" {
+		proxyUrl, err := url.Parse(os.Getenv("LOCAL_PROXY"))
+		if err == nil {
+			tr := http.Transport{
+				Proxy: http.ProxyURL(proxyUrl),
+			}
+			hc.Transport = &tr
+		}
+	}
+
+	oc := OpenAICompatible{
+		ApiKey:     apikey,
+		Client:     hc,
+		tokenUsage: &llm.TokenUsage{},
+		Done:       make(chan struct{}),
+	}
+
+	if apikey.Parameters != nil {
+		var params map[string]interface{}
+		err := json.Unmarshal([]byte(*apikey.Parameters), &params)
+		if err != nil {
+			return nil, err
+		}
+		oc.Params = params
+	}
+
+	return &oc, nil
+}
+
+func (o *OpenAICompatible) Chat(ctx context.Context, chatReq llm.ChatRequest) (*llm.ChatResponse, error) {
+	chatReq.Stream = false
+	dst, err := utils.StructToMap(chatReq)
+	if err != nil {
+		return nil, err
+	}
+	if len(o.Params) > 0 {
+		dst = utils.MergeJSONObjects(dst, o.Params)
+	}
+
+	var reqBody bytes.Buffer
+	if err := json.NewEncoder(&reqBody).Encode(dst); err != nil {
+		return nil, err
+	}
+
+	var req *http.Request
+	switch *o.ApiKey.ApiType {
+	case "azure":
+		formatModel := func(in string) string {
+			if strings.Contains(in, ".") {
+				return strings.ReplaceAll(in, ".", "")
+			}
+			return in
+		}
+		var buildurl string
+		if *o.ApiKey.Endpoint != "" {
+			buildurl = fmt.Sprintf("%s/openai/deployments/%s/chat/completions?api-version=%s", *o.ApiKey.Endpoint, formatModel(chatReq.Model), AzureApiVersion)
+		} else {
+			buildurl = fmt.Sprintf("https://%s.openai.azure.com/openai/deployments/%s/chat/completions?api-version=%s", *o.ApiKey.ResourceNmae, formatModel(chatReq.Model), AzureApiVersion)
+		}
+		req, _ = http.NewRequest(http.MethodPost, buildurl, &reqBody)
+		req.Header.Set("api-key", *o.ApiKey.ApiKey)
+	case "github":
+		req, _ = http.NewRequest(http.MethodPost, Github_Marketplace, &reqBody)
+	default:
+		if o.ApiKey.Endpoint == nil || *o.ApiKey.Endpoint == "" {
+			req, _ = http.NewRequest(http.MethodPost, defaultOpenAICompatibleEndpoint, &reqBody)
+		} else {
+			req, _ = http.NewRequest(http.MethodPost, *o.ApiKey.Endpoint, &reqBody)
+		}
+	}
+	req.Header.Set("Authorization", "Bearer "+*o.ApiKey.ApiKey)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept-Encoding", "identity")
+
+	resp, err := o.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var chatResp llm.ChatResponse
+	if err := json.NewDecoder(resp.Body).Decode(&chatResp); err != nil {
+		return nil, err
+	}
+
+	o.tokenUsage.PromptTokens = chatResp.Usage.PromptTokens
+	o.tokenUsage.CompletionTokens = chatResp.Usage.CompletionTokens
+	o.tokenUsage.TotalTokens = chatResp.Usage.TotalTokens
+	return &chatResp, nil
+}
+
+func (o *OpenAICompatible) StreamChat(ctx context.Context, chatReq llm.ChatRequest) (chan *llm.StreamChatResponse, error) {
+	chatReq.Stream = true
+	dst, err := utils.StructToMap(chatReq)
+	if err != nil {
+		return nil, err
+	}
+	if len(o.Params) > 0 {
+		dst = utils.MergeJSONObjects(dst, o.Params)
+	}
+
+	var reqBody bytes.Buffer
+	if err := json.NewEncoder(&reqBody).Encode(dst); err != nil {
+		return nil, err
+	}
+
+	var req *http.Request
+	switch *o.ApiKey.ApiType {
+	case "azure":
+		formatModel := func(in string) string {
+			if strings.Contains(in, ".") {
+				return strings.ReplaceAll(in, ".", "")
+			}
+			return in
+		}
+		var buildurl string
+		if *o.ApiKey.Endpoint != "" {
+			buildurl = fmt.Sprintf("%s/openai/deployments/%s/chat/completions?api-version=%s", *o.ApiKey.Endpoint, formatModel(chatReq.Model), AzureApiVersion)
+		} else {
+			buildurl = fmt.Sprintf("https://%s.openai.azure.com/openai/deployments/%s/chat/completions?api-version=%s", *o.ApiKey.ResourceNmae, formatModel(chatReq.Model), AzureApiVersion)
+		}
+		req, _ = http.NewRequest(http.MethodPost, buildurl, &reqBody)
+		req.Header.Set("api-key", *o.ApiKey.ApiKey)
+	case "github":
+		req, _ = http.NewRequest(http.MethodPost, Github_Marketplace, &reqBody)
+	default:
+		if o.ApiKey.Endpoint == nil || *o.ApiKey.Endpoint == "" {
+			req, _ = http.NewRequest(http.MethodPost, defaultOpenAICompatibleEndpoint, &reqBody)
+		} else {
+			req, _ = http.NewRequest(http.MethodPost, *o.ApiKey.Endpoint, &reqBody)
+		}
+	}
+	req.Header.Set("Authorization", "Bearer "+*o.ApiKey.ApiKey)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept-Encoding", "identity")
+
+	resp, err := o.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	output := make(chan *llm.StreamChatResponse)
+
+	b := new(bytes.Buffer)
+	teeReader := io.TeeReader(resp.Body, b)
+	// 流式响应
+	scanner := bufio.NewScanner(teeReader)
+
+	go func() {
+		defer resp.Body.Close()
+		defer close(output)
+
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			var streamResp llm.StreamChatResponse
+			if len(line) > 0 {
+				// fmt.Println(string(line))
+				if bytes.HasPrefix(line, []byte("data: ")) {
+					if bytes.HasPrefix(line, []byte("data: [DONE]")) {
+						break
+					}
+					line = bytes.Replace(line, []byte("data: "), []byte(""), -1)
+					line = bytes.TrimSpace(line)
+					if err := json.Unmarshal(line, &streamResp); err != nil {
+						continue
+					}
+					if streamResp.Usage != nil {
+						o.tokenUsage.PromptTokens += streamResp.Usage.PromptTokens
+						o.tokenUsage.CompletionTokens += streamResp.Usage.CompletionTokens
+						o.tokenUsage.TotalTokens += streamResp.Usage.TotalTokens
+					}
+					output <- &streamResp
+				}
+			}
+
+			// select {
+			// case <-ctx.Done():
+			// 	return
+			// case output <- &streamResp:
+			// }
+		}
+	}()
+	return output, nil
+}
+
+func (o *OpenAICompatible) GetTokenUsage() *llm.TokenUsage {
+	return o.tokenUsage
+}
--- a/llm/types.go
+++ b/llm/types.go
@@ -0,0 +1,41 @@
+package llm
+
+import (
+	"fmt"
+
+	"github.com/sashabaranov/go-openai"
+)
+
+type ChatRequest openai.ChatCompletionRequest
+
+type ChatResponse openai.ChatCompletionResponse
+
+type StreamChatResponse openai.ChatCompletionStreamResponse
+
+type ChatMessage openai.ChatCompletionMessage
+
+type TokenUsage struct {
+	PromptTokens     int `json:"prompt_tokens"`
+	CompletionTokens int `json:"completion_tokens"`
+	ToolsTokens      int `json:"total_tokens"`
+	TotalTokens      int `json:"total_tokens"`
+}
+
+type ErrorResponse struct {
+	Err struct {
+		Message string `json:"message,omitempty"`
+		Type    string `json:"type,omitempty"`
+		Param   string `json:"param,omitempty"`
+		Code    string `json:"code,omitempty"`
+	} `json:"error,omitempty"`
+	HTTPStatusCode int    `json:"-"`
+	HTTPStatus     string `json:"-"`
+}
+
+func (e ErrorResponse) Error() string {
+	if e.HTTPStatusCode > 0 {
+		return fmt.Sprintf("error, status code: %d, status: %s, message: %s", e.HTTPStatusCode, e.HTTPStatus, e.Err.Message)
+	}
+
+	return e.Err.Message
+}
--- a/llm/vertexai/auth.go
+++ b/llm/vertexai/auth.go
@@ -0,0 +1,186 @@
+/*
+https://docs.anthropic.com/zh-CN/api/claude-on-vertex-ai
+
+MODEL_ID=claude-3-5-sonnet@20240620
+REGION=us-east5
+PROJECT_ID=MY_PROJECT_ID
+
+curl \
+-X POST \
+-H "Authorization: Bearer $(gcloud auth print-access-token)" \
+-H "Content-Type: application/json" \
+https://$LOCATION-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/anthropic/models/${MODEL_ID}:streamRawPredict \
+-d '{
+  "anthropic_version": "vertex-2023-10-16",
+  "messages": [{
+    "role": "user",
+    "content": "介绍一下你自己"
+  }],
+  "stream": true,
+  "max_tokens": 4096
+}'
+
+quota:
+https://console.cloud.google.com/iam-admin/quotas?hl=zh-cn
+*/
+
+package vertexai
+
+import (
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+)
+
+// json文件存储在ApiKey.ApiSecret中
+type VertexSecretKey struct {
+	Type                    string `json:"type"`
+	ProjectID               string `json:"project_id"`
+	PrivateKeyID            string `json:"private_key_id"`
+	PrivateKey              string `json:"private_key"`
+	ClientEmail             string `json:"client_email"`
+	ClientID                string `json:"client_id"`
+	AuthURI                 string `json:"auth_uri"`
+	TokenURI                string `json:"token_uri"`
+	AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"`
+	ClientX509CertURL       string `json:"client_x509_cert_url"`
+	UniverseDomain          string `json:"universe_domain"`
+}
+
+type VertexClaudeModel struct {
+	VertexName string
+	Region     string
+}
+
+var VertexClaudeModelMap = map[string]VertexClaudeModel{
+	"claude-3-opus": {
+		VertexName: "claude-3-opus@20240229",
+		Region:     "us-east5",
+	},
+	"claude-3-sonnet": {
+		VertexName: "claude-3-sonnet@20240229",
+		Region:     "us-central1",
+		// Region:     "asia-southeast1",
+	},
+	"claude-3-haiku": {
+		VertexName: "claude-3-haiku@20240307",
+		Region:     "us-central1",
+		// Region:     "europe-west4",
+	},
+	"claude-3-opus-20240229": {
+		VertexName: "claude-3-opus@20240229",
+		Region:     "us-east5",
+	},
+	"claude-3-sonnet-20240229": {
+		VertexName: "claude-3-sonnet@20240229",
+		Region:     "us-central1",
+		// Region:     "asia-southeast1",
+	},
+	"claude-3-haiku-20240307": {
+		VertexName: "claude-3-haiku@20240307",
+		Region:     "us-central1",
+		// Region:     "europe-west4",
+	},
+	"claude-3-5-sonnet": {
+		VertexName: "claude-3-5-sonnet@20240620",
+		Region:     "us-east5",
+		// Region:     "europe-west1",
+	},
+	"claude-3-5-sonnet-20240620": {
+		VertexName: "claude-3-5-sonnet@20240620",
+		Region:     "us-east5",
+		// Region:     "europe-west1",
+	},
+	"claude-3-5-sonnet-20241022": {
+		VertexName: "claude-3-5-sonnet-v2@20241022",
+		Region:     "us-east5",
+	},
+	"claude-3-5-sonnet-latest": { //可能没有容量,指向老模型
+		VertexName: "claude-3-5-sonnet@20240620",
+		Region:     "us-east5",
+	},
+}
+
+func createSignedJWT(email, privateKeyPEM string) (string, error) {
+	block, _ := pem.Decode([]byte(privateKeyPEM))
+	if block == nil {
+		return "", fmt.Errorf("failed to parse PEM block containing the private key")
+	}
+
+	privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return "", err
+	}
+
+	rsaKey, ok := privateKey.(*rsa.PrivateKey)
+	if !ok {
+		return "", fmt.Errorf("not an RSA private key")
+	}
+
+	now := time.Now()
+	claims := jwt.MapClaims{
+		"iss":   email,
+		"aud":   "https://www.googleapis.com/oauth2/v4/token",
+		"iat":   now.Unix(),
+		"exp":   now.Add(10 * time.Minute).Unix(),
+		"scope": "https://www.googleapis.com/auth/cloud-platform",
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
+	return token.SignedString(rsaKey)
+}
+
+func exchangeJwtForAccessToken(signedJWT string) (string, error) {
+	authURL := "https://www.googleapis.com/oauth2/v4/token"
+	data := url.Values{}
+	data.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
+	data.Set("assertion", signedJWT)
+
+	client := http.DefaultClient
+	if os.Getenv("LOCAL_PROXY") != "" {
+		if proxyUrl, err := url.Parse(os.Getenv("LOCAL_PROXY")); err == nil {
+			client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
+		}
+	}
+
+	resp, err := client.PostForm(authURL, data)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	var result map[string]interface{}
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return "", err
+	}
+
+	accessToken, ok := result["access_token"].(string)
+	if !ok {
+		return "", fmt.Errorf("access token not found in response")
+	}
+
+	return accessToken, nil
+}
+
+// 获取gcloud auth token
+func GcloudAuth(ClientEmail, PrivateKey string) (string, error) {
+	signedJWT, err := createSignedJWT(ClientEmail, PrivateKey)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := exchangeJwtForAccessToken(signedJWT)
+	if err != nil {
+		return "", fmt.Errorf("Invalid jwt token: %v\n", err)
+	}
+
+	return token, nil
+}