mirrors2/opencatd-open
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reface to openteam

Browse Source
This commit is contained in:
Sakurasan
2025-04-16 18:01:27 +08:00
parent bc223d6530
commit e7ffc9e8b9
92 changed files with 5345 additions and 1273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
llm/claude/chat.go Normal file
View File

@@ -0,0 +1,138 @@
// https://docs.anthropic.com/claude/reference/messages_post
package claude
import (
"context"
"encoding/json"
"opencatd-open/internal/model"
"opencatd-open/llm"
"opencatd-open/llm/openai"
"github.com/gin-gonic/gin"
)
func ChatProxy(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
ChatMessages(c, chatReq)
}
func ChatTextCompletions(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
}
type ChatRequest struct {
Model string `json:"model,omitempty"`
Messages any `json:"messages,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
Stream bool `json:"stream,omitempty"`
System string `json:"system,omitempty"`
TopK int `json:"top_k,omitempty"`
TopP float64 `json:"top_p,omitempty"`
Temperature float64 `json:"temperature,omitempty"`
AnthropicVersion string `json:"anthropic_version,omitempty"`
}
func (c *ChatRequest) ByteJson() []byte {
bytejson, _ := json.Marshal(c)
return bytejson
}
type ChatMessage struct {
Role string `json:"role,omitempty"`
Content string `json:"content,omitempty"`
}
type VisionMessages struct {
Role string `json:"role,omitempty"`
Content []VisionContent `json:"content,omitempty"`
}
type VisionContent struct {
Type string `json:"type,omitempty"`
Source *VisionSource `json:"source,omitempty"`
Text string `json:"text,omitempty"`
}
type VisionSource struct {
Type string `json:"type,omitempty"`
MediaType string `json:"media_type,omitempty"`
Data string `json:"data,omitempty"`
}
type ChatResponse struct {
ID string `json:"id"`
Type string `json:"type"`
Role string `json:"role"`
Model string `json:"model"`
StopSequence any `json:"stop_sequence"`
Usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
} `json:"usage"`
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content"`
StopReason string `json:"stop_reason"`
}
type ClaudeStreamResponse struct {
Type string `json:"type"`
Index int `json:"index"`
ContentBlock struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content_block"`
Delta struct {
Type string `json:"type"`
Text string `json:"text"`
StopReason string `json:"stop_reason"`
StopSequence any `json:"stop_sequence"`
} `json:"delta"`
Message struct {
ID string `json:"id"`
Type string `json:"type"`
Role string `json:"role"`
Content []any `json:"content"`
Model string `json:"model"`
StopReason string `json:"stop_reason"`
StopSequence any `json:"stop_sequence"`
Usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
} `json:"usage"`
} `json:"message"`
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
Usage struct {
OutputTokens int `json:"output_tokens"`
} `json:"usage"`
}
type Claude struct {
Ctx context.Context
ApiKey *model.ApiKey
tokenUsage *llm.TokenUsage
Done chan struct{}
}
func NewClaude(ctx context.Context, apiKey *model.ApiKey) (*Claude, error) {
return &Claude{
Ctx: context.Background(),
ApiKey: apiKey,
tokenUsage: &llm.TokenUsage{},
Done: make(chan struct{}),
}, nil
}
func (c *Claude) Chat(ctx context.Context, chatReq llm.ChatRequest) (*llm.ChatResponse, error) {
return nil, nil
}
func (g *Claude) StreamChat(ctx context.Context, chatReq llm.ChatRequest) (chan *llm.StreamChatResponse, error) {
return nil, nil
}

429
llm/claude/claude.go Normal file
View File

@@ -0,0 +1,429 @@
/*
https://docs.anthropic.com/claude/reference/complete_post
curl --request POST \
--url https://api.anthropic.com/v1/complete \
--header "anthropic-version: 2023-06-01" \
--header "content-type: application/json" \
--header "x-api-key: $ANTHROPIC_API_KEY" \
--data '
{
"model": "claude-2",
"prompt": "\n\nHuman: Hello, world!\n\nAssistant:",
"max_tokens_to_sample": 256,
"stream": true
}
'
{"completion":" Hello! Nice to meet you.","stop_reason":"stop_sequence","model":"claude-2.0","stop":"\n\nHuman:","log_id":"727bded01002627057967d02b3d557a01aa73266849b62f5aa0b97dec1247ed3"}
event: completion
data: {"completion":"","stop_reason":"stop_sequence","model":"claude-2.0","stop":"\n\nHuman:","log_id":"dfd42341ad08856ff01811885fb8640a1bf977551d8331f81fe9a6c8182c6c63"}
# Model Pricing
Claude Instant |100,000 tokens |Prompt $1.63/million tokens |Completion $5.51/million tokens
Claude 2 |100,000 tokens |Prompt $11.02/million tokens |Completion $32.68/million tokens
*Claude 1 is still accessible and offered at the same price as Claude 2.
# AWS
https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-service.html
https://aws.amazon.com/cn/bedrock/pricing/
Anthropic models Price for 1000 input tokens Price for 1000 output tokens
Claude Instant $0.00163 $0.00551
Claude $0.01102 $0.03268
https://docs.aws.amazon.com/bedrock/latest/userguide/endpointsTable.html
地区名称 地区 端点 协议
美国东部（弗吉尼亚北部） 美国东部1 bedrock-runtime.us-east-1.amazonaws.com HTTPS
bedrock-runtime-fips.us-east-1.amazonaws.com HTTPS
美国西部（俄勒冈州） 美国西2号 bedrock-runtime.us-west-2.amazonaws.com HTTPS
bedrock-runtime-fips.us-west-2.amazonaws.com HTTPS
亚太地区（新加坡） ap-东南-1 bedrock-runtime.ap-southeast-1.amazonaws.com HTTPS
*/
// package anthropic
package claude
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/http/httputil"
"net/url"
"opencatd-open/pkg/tokenizer"
"opencatd-open/store"
"strings"
"sync"
"time"
"github.com/gin-gonic/gin"
"github.com/sashabaranov/go-openai"
)
var (
ClaudeUrl = "https://api.anthropic.com/v1/complete"
ClaudeMessageEndpoint = "https://api.anthropic.com/v1/messages"
)
type MessageModule struct {
Assistant string // returned data (do not modify)
Human string // input content
}
type CompleteRequest struct {
Model string `json:"model,omitempty"` //*
Prompt string `json:"prompt,omitempty"` //*
MaxTokensToSample int `json:"max_tokens_to_sample,omitempty"` //*
StopSequences string `json:"stop_sequences,omitempty"`
Temperature int `json:"temperature,omitempty"`
TopP int `json:"top_p,omitempty"`
TopK int `json:"top_k,omitempty"`
Stream bool `json:"stream,omitempty"`
Metadata struct {
UserId string `json:"user_Id,omitempty"`
} `json:"metadata,omitempty"`
}
type CompleteResponse struct {
Completion string `json:"completion"`
StopReason string `json:"stop_reason"`
Model string `json:"model"`
Stop string `json:"stop"`
LogID string `json:"log_id"`
}
func Create() {
complet := CompleteRequest{
Model: "claude-2",
Prompt: "Human: Hello, world!\\n\\nAssistant:",
Stream: true,
}
var payload *bytes.Buffer
json.NewEncoder(payload).Encode(complet)
// payload := strings.NewReader("{\"model\":\"claude-2\",\"prompt\":\"\\n\\nHuman: Hello, world!\\n\\nAssistant:\",\"max_tokens_to_sample\":256}")
req, _ := http.NewRequest("POST", ClaudeUrl, payload)
req.Header.Add("accept", "application/json")
req.Header.Add("anthropic-version", "2023-06-01")
req.Header.Add("x-api-key", "$ANTHROPIC_API_KEY")
req.Header.Add("content-type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
// body, _ := io.ReadAll(res.Body)
// fmt.Println(string(body))
reader := bufio.NewReader(res.Body)
for {
line, err := reader.ReadString('\n')
if err == nil {
if strings.HasPrefix(line, "data:") {
fmt.Println(line)
// var result CompleteResponse
// json.Unmarshal()
} else {
continue
}
} else {
break
}
}
}
func ClaudeProxy(c *gin.Context) {
var chatlog store.Tokens
var complete CompleteRequest
byteBody, _ := io.ReadAll(c.Request.Body)
c.Request.Body = io.NopCloser(bytes.NewBuffer(byteBody))
if err := json.Unmarshal(byteBody, &complete); err != nil {
c.AbortWithError(http.StatusBadRequest, err)
return
}
key, err := store.SelectKeyCache("claude") //anthropic
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": gin.H{
"message": err.Error(),
},
})
return
}
chatlog.Model = complete.Model
token, _ := c.Get("localuser")
lu, err := store.GetUserByToken(token.(string))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": gin.H{
"message": err.Error(),
},
})
return
}
chatlog.UserID = int(lu.ID)
chatlog.PromptCount = tokenizer.NumTokensFromStr(complete.Prompt, complete.Model)
if key.EndPoint == "" {
key.EndPoint = "https://api.anthropic.com"
}
targetUrl, _ := url.ParseRequestURI(key.EndPoint + c.Request.URL.String())
proxy := httputil.NewSingleHostReverseProxy(targetUrl)
proxy.Director = func(req *http.Request) {
req.Host = targetUrl.Host
req.URL.Scheme = targetUrl.Scheme
req.URL.Host = targetUrl.Host
req.Header.Set("anthropic-version", "2023-06-01")
req.Header.Set("content-type", "application/json")
req.Header.Set("x-api-key", key.Key)
}
proxy.ModifyResponse = func(resp *http.Response) error {
if resp.StatusCode != http.StatusOK {
return nil
}
var byteResp []byte
byteResp, _ = io.ReadAll(resp.Body)
resp.Body = io.NopCloser(bytes.NewBuffer(byteResp))
if complete.Stream != true {
var complete_resp CompleteResponse
if err := json.Unmarshal(byteResp, &complete_resp); err != nil {
log.Println(err)
return nil
}
chatlog.CompletionCount = tokenizer.NumTokensFromStr(complete_resp.Completion, chatlog.Model)
} else {
var completion string
for {
line, err := bufio.NewReader(bytes.NewBuffer(byteResp)).ReadString('\n')
if err != nil {
if strings.HasPrefix(line, "data:") {
line = strings.TrimSpace(strings.TrimPrefix(line, "data:"))
if strings.HasSuffix(line, "[DONE]") {
break
}
line = strings.TrimSpace(line)
var complete_resp CompleteResponse
if err := json.Unmarshal([]byte(line), &complete_resp); err != nil {
log.Println(err)
break
}
completion += line
}
}
}
log.Println("completion:", completion)
chatlog.CompletionCount = tokenizer.NumTokensFromStr(completion, chatlog.Model)
}
// calc cost
chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
if err := store.Record(&chatlog); err != nil {
log.Println(err)
}
if err := store.SumDaily(chatlog.UserID); err != nil {
log.Println(err)
}
return nil
}
proxy.ServeHTTP(c.Writer, c.Request)
}
func TransReq(chatreq *openai.ChatCompletionRequest) (*bytes.Buffer, error) {
transReq := CompleteRequest{
Model: chatreq.Model,
Temperature: int(chatreq.Temperature),
TopP: int(chatreq.TopP),
Stream: chatreq.Stream,
MaxTokensToSample: chatreq.MaxTokens,
}
if transReq.MaxTokensToSample == 0 {
transReq.MaxTokensToSample = 100000
}
var prompt string
for _, msg := range chatreq.Messages {
switch msg.Role {
case "system":
prompt += fmt.Sprintf("\n\nHuman:%s", msg.Content)
case "user":
prompt += fmt.Sprintf("\n\nHuman:%s", msg.Content)
case "assistant":
prompt += fmt.Sprintf("\n\nAssistant:%s", msg.Content)
}
}
transReq.Prompt = prompt + "\n\nAssistant:"
var payload = bytes.NewBuffer(nil)
if err := json.NewEncoder(payload).Encode(transReq); err != nil {
return nil, err
}
return payload, nil
}
func TransRsp(c *gin.Context, isStream bool, chatlog store.Tokens, reader *bufio.Reader) {
if !isStream {
var completersp CompleteResponse
var chatrsp openai.ChatCompletionResponse
json.NewDecoder(reader).Decode(&completersp)
chatrsp.Model = completersp.Model
chatrsp.ID = completersp.LogID
chatrsp.Object = "chat.completion"
chatrsp.Created = time.Now().Unix()
choice := openai.ChatCompletionChoice{
Index: 0,
FinishReason: "stop",
Message: openai.ChatCompletionMessage{
Role: "assistant",
Content: completersp.Completion,
},
}
chatrsp.Choices = append(chatrsp.Choices, choice)
var payload *bytes.Buffer
if err := json.NewEncoder(payload).Encode(chatrsp); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": gin.H{
"message": err.Error(),
},
})
return
}
chatlog.CompletionCount = tokenizer.NumTokensFromStr(completersp.Completion, chatlog.Model)
chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
if err := store.Record(&chatlog); err != nil {
log.Println(err)
}
if err := store.SumDaily(chatlog.UserID); err != nil {
log.Println(err)
}
c.JSON(http.StatusOK, payload)
return
} else {
var (
wg sync.WaitGroup
dataChan = make(chan string)
stopChan = make(chan bool)
complete_resp string
)
wg.Add(2)
go func() {
defer wg.Done()
for {
line, err := reader.ReadString('\n')
if err == nil {
if strings.HasPrefix(line, "data: ") {
var result CompleteResponse
json.NewDecoder(strings.NewReader(line[6:])).Decode(&result)
if result.StopReason == "" {
if result.Completion != "" {
complete_resp += result.Completion
chatrsp := openai.ChatCompletionStreamResponse{
ID: result.LogID,
Model: result.Model,
Object: "chat.completion",
Created: time.Now().Unix(),
}
choice := openai.ChatCompletionStreamChoice{
Delta: openai.ChatCompletionStreamChoiceDelta{
Role: "assistant",
Content: result.Completion,
},
FinishReason: "",
}
chatrsp.Choices = append(chatrsp.Choices, choice)
bytedate, _ := json.Marshal(chatrsp)
dataChan <- string(bytedate)
}
} else {
chatrsp := openai.ChatCompletionStreamResponse{
ID: result.LogID,
Model: result.Model,
Object: "chat.completion",
Created: time.Now().Unix(),
}
choice := openai.ChatCompletionStreamChoice{
Delta: openai.ChatCompletionStreamChoiceDelta{
Role: "assistant",
Content: result.Completion,
},
}
choice.FinishReason = openai.FinishReason(TranslatestopReason(result.StopReason))
chatrsp.Choices = append(chatrsp.Choices, choice)
bytedate, _ := json.Marshal(chatrsp)
dataChan <- string(bytedate)
dataChan <- "[DONE]"
break
}
} else {
continue
}
} else {
break
}
}
close(dataChan)
stopChan <- true
close(stopChan)
}()
go func() {
defer wg.Done()
Loop:
for {
select {
case data := <-dataChan:
if data != "" {
c.Writer.WriteString("data: " + data)
c.Writer.WriteString("\n\n")
c.Writer.Flush()
}
case <-stopChan:
break Loop
}
}
}()
wg.Wait()
chatlog.CompletionCount = tokenizer.NumTokensFromStr(complete_resp, chatlog.Model)
chatlog.TotalTokens = chatlog.PromptCount + chatlog.CompletionCount
chatlog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(chatlog.Model, chatlog.PromptCount, chatlog.CompletionCount))
if err := store.Record(&chatlog); err != nil {
log.Println(err)
}
if err := store.SumDaily(chatlog.UserID); err != nil {
log.Println(err)
}
}
}
// claude -> openai
func TranslatestopReason(reason string) string {
switch reason {
case "stop_sequence":
return "stop"
case "max_tokens":
return "length"
default:
return reason
}
}

230
llm/claude/handle_proxy.go Normal file
View File

@@ -0,0 +1,230 @@
package claude
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"opencatd-open/llm/openai"
"opencatd-open/llm/vertexai"
"opencatd-open/pkg/error"
"opencatd-open/pkg/tokenizer"
"opencatd-open/store"
"strings"
"github.com/gin-gonic/gin"
)
func ChatMessages(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
var (
req *http.Request
targetURL = ClaudeMessageEndpoint
)
apiKey, err := store.SelectKeyCache("claude")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
usagelog := store.Tokens{Model: chatReq.Model}
var claudReq ChatRequest
claudReq.Model = chatReq.Model
claudReq.Stream = chatReq.Stream
// claudReq.Temperature = chatReq.Temperature
claudReq.TopP = chatReq.TopP
claudReq.MaxTokens = 4096
if apiKey.ApiType == "vertex" {
claudReq.AnthropicVersion = "vertex-2023-10-16"
claudReq.Model = ""
}
var claudecontent []VisionContent
var prompt string
for _, msg := range chatReq.Messages {
switch ct := msg.Content.(type) {
case string:
prompt += "<" + msg.Role + ">: " + msg.Content.(string) + "\n"
if msg.Role == "system" {
claudReq.System = msg.Content.(string)
continue
}
claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + msg.Content.(string)})
case []any:
for _, item := range ct {
if m, ok := item.(map[string]interface{}); ok {
if m["type"] == "text" {
prompt += "<" + msg.Role + ">: " + m["text"].(string) + "\n"
claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + m["text"].(string)})
} else if m["type"] == "image_url" {
if url, ok := m["image_url"].(map[string]interface{}); ok {
fmt.Printf(" URL: %v\n", url["url"])
if strings.HasPrefix(url["url"].(string), "http") {
fmt.Println("网络图片:", url["url"].(string))
} else if strings.HasPrefix(url["url"].(string), "data:image") {
fmt.Println("base64:", url["url"].(string)[:20])
var mediaType string
if strings.HasPrefix(url["url"].(string), "data:image/jpeg") {
mediaType = "image/jpeg"
}
if strings.HasPrefix(url["url"].(string), "data:image/png") {
mediaType = "image/png"
}
claudecontent = append(claudecontent, VisionContent{Type: "image", Source: &VisionSource{Type: "base64", MediaType: mediaType, Data: strings.Split(url["url"].(string), ",")[1]}})
}
}
}
}
}
default:
c.JSON(http.StatusInternalServerError, gin.H{
"error": gin.H{
"message": "Invalid content type",
},
})
return
}
if len(chatReq.Tools) > 0 {
tooljson, _ := json.Marshal(chatReq.Tools)
prompt += "<tools>: " + string(tooljson) + "\n"
}
}
claudReq.Messages = []VisionMessages{{Role: "user", Content: claudecontent}}
usagelog.PromptCount = tokenizer.NumTokensFromStr(prompt, chatReq.Model)
if apiKey.ApiType == "vertex" {
var vertexSecret vertexai.VertexSecretKey
if err := json.Unmarshal([]byte(apiKey.ApiSecret), &vertexSecret); err != nil {
c.JSON(http.StatusInternalServerError, error.ErrorData(err.Error()))
return
}
vcmodel, ok := vertexai.VertexClaudeModelMap[chatReq.Model]
if !ok {
c.JSON(http.StatusInternalServerError, error.ErrorData("Model not found"))
return
}
// 获取gcloud token，临时放置在apiKey.Key中
gcloudToken, err := vertexai.GcloudAuth(vertexSecret.ClientEmail, vertexSecret.PrivateKey)
if err != nil {
c.JSON(http.StatusInternalServerError, error.ErrorData(err.Error()))
return
}
// 拼接vertex的请求地址
targetURL = fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1/projects/%s/locations/%s/publishers/anthropic/models/%s:streamRawPredict", vcmodel.Region, vertexSecret.ProjectID, vcmodel.Region, vcmodel.VertexName)
req, _ = http.NewRequest("POST", targetURL, bytes.NewReader(claudReq.ByteJson()))
req.Header.Set("Authorization", "Bearer "+gcloudToken)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "text/event-stream")
req.Header.Set("Accept-Encoding", "identity")
} else {
req, _ = http.NewRequest("POST", targetURL, bytes.NewReader(claudReq.ByteJson()))
req.Header.Set("x-api-key", apiKey.Key)
req.Header.Set("anthropic-version", "2023-06-01")
req.Header.Set("Content-Type", "application/json")
}
client := http.DefaultClient
rsp, err := client.Do(req)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
defer rsp.Body.Close()
if rsp.StatusCode != http.StatusOK {
io.Copy(c.Writer, rsp.Body)
return
}
var buffer bytes.Buffer
teeReader := io.TeeReader(rsp.Body, &buffer)
dataChan := make(chan string)
// stopChan := make(chan bool)
var result string
scanner := bufio.NewScanner(teeReader)
go func() {
for scanner.Scan() {
line := scanner.Bytes()
if len(line) > 0 && bytes.HasPrefix(line, []byte("data: ")) {
if bytes.HasPrefix(line, []byte("data: [DONE]")) {
dataChan <- string(line) + "\n"
break
}
var claudeResp ClaudeStreamResponse
line = bytes.Replace(line, []byte("data: "), []byte(""), -1)
line = bytes.TrimSpace(line)
if err := json.Unmarshal(line, &claudeResp); err != nil {
continue
}
if claudeResp.Type == "message_start" {
if claudeResp.Message.Role != "" {
result += "<" + claudeResp.Message.Role + ">"
}
} else if claudeResp.Type == "message_stop" {
break
}
if claudeResp.Delta.Text != "" {
result += claudeResp.Delta.Text
}
var choice openai.Choice
choice.Delta.Role = claudeResp.Message.Role
choice.Delta.Content = claudeResp.Delta.Text
choice.FinishReason = claudeResp.Delta.StopReason
chatResp := openai.ChatCompletionStreamResponse{
Model: chatReq.Model,
Choices: []openai.Choice{choice},
}
dataChan <- "data: " + string(chatResp.ByteJson()) + "\n"
if claudeResp.Delta.StopReason != "" {
dataChan <- "\ndata: [DONE]\n"
}
}
}
defer close(dataChan)
}()
c.Writer.Header().Set("Content-Type", "text/event-stream")
c.Writer.Header().Set("Cache-Control", "no-cache")
c.Writer.Header().Set("Connection", "keep-alive")
c.Writer.Header().Set("Transfer-Encoding", "chunked")
c.Writer.Header().Set("X-Accel-Buffering", "no")
c.Stream(func(w io.Writer) bool {
if data, ok := <-dataChan; ok {
if strings.HasPrefix(data, "data: ") {
c.Writer.WriteString(data)
// c.Writer.WriteString("\n\n")
} else {
c.Writer.WriteHeader(http.StatusBadGateway)
c.Writer.WriteString(data)
}
c.Writer.Flush()
return true
}
go func() {
usagelog.CompletionCount = tokenizer.NumTokensFromStr(result, chatReq.Model)
usagelog.Cost = fmt.Sprintf("%.6f", tokenizer.Cost(usagelog.Model, usagelog.PromptCount, usagelog.CompletionCount))
if err := store.Record(&usagelog); err != nil {
log.Println(err)
}
if err := store.SumDaily(usagelog.UserID); err != nil {
log.Println(err)
}
}()
return false
})
}

246
llm/claude/v2/chat.go Normal file
View File

@@ -0,0 +1,246 @@
package claude
import (
"context"
"encoding/base64"
"net/http"
"net/url"
"opencatd-open/internal/model"
"opencatd-open/llm"
"os"
"strings"
"github.com/liushuangls/go-anthropic/v2"
"github.com/sashabaranov/go-openai"
)
type Claude struct {
Ctx context.Context
ApiKey *model.ApiKey
tokenUsage *llm.TokenUsage
Done chan struct{}
Client *anthropic.Client
}
func NewClaude(apiKey *model.ApiKey) (*Claude, error) {
opts := []anthropic.ClientOption{}
if os.Getenv("LOCAL_PROXY") != "" {
proxyUrl, err := url.Parse(os.Getenv("LOCAL_PROXY"))
if err == nil {
client := http.DefaultClient
client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
opts = append(opts, anthropic.WithHTTPClient(client))
}
}
return &Claude{
Ctx: context.Background(),
ApiKey: apiKey,
tokenUsage: &llm.TokenUsage{},
Done: make(chan struct{}),
Client: anthropic.NewClient(*apiKey.ApiKey, opts...),
}, nil
}
func (c *Claude) Chat(ctx context.Context, chatReq llm.ChatRequest) (*llm.ChatResponse, error) {
var messages []anthropic.Message
if len(chatReq.Messages) > 0 {
for _, msg := range chatReq.Messages {
var role anthropic.ChatRole
if msg.Role != "assistant" {
role = anthropic.RoleUser
} else {
role = anthropic.RoleAssistant
}
var content []anthropic.MessageContent
if len(msg.MultiContent) > 0 {
for _, mc := range msg.MultiContent {
if mc.Type == "text" {
content = append(content, anthropic.MessageContent{Type: anthropic.MessagesContentTypeText, Text: &mc.Text})
}
if mc.Type == "image_url" {
if strings.HasPrefix(mc.ImageURL.URL, "http") {
continue
}
if strings.HasPrefix(mc.ImageURL.URL, "data:image") {
var mediaType string
if strings.HasPrefix(mc.ImageURL.URL, "data:image/jpeg") {
mediaType = "image/jpeg"
}
if strings.HasPrefix(mc.ImageURL.URL, "data:image/png") {
mediaType = "image/png"
}
imageString := strings.Split(mc.ImageURL.URL, ",")[1]
imageBytes, _ := base64.StdEncoding.DecodeString(imageString)
content = append(content, anthropic.MessageContent{Type: "image", Source: &anthropic.MessageContentSource{Type: "base64", MediaType: mediaType, Data: imageBytes}})
}
}
messages = append(messages, anthropic.Message{Role: role, Content: content})
}
} else {
if len(msg.Content) > 0 {
content = append(content, anthropic.MessageContent{Type: "text", Text: &msg.Content})
}
}
messages = append(messages, anthropic.Message{Role: role, Content: content})
}
}
var maxTokens int
if chatReq.MaxTokens > 0 {
maxTokens = chatReq.MaxTokens
} else {
if strings.Contains(chatReq.Model, "sonnet") || strings.Contains(chatReq.Model, "haiku") {
maxTokens = 8192
} else {
maxTokens = 4096
}
}
resp, err := c.Client.CreateMessages(ctx, anthropic.MessagesRequest{
Model: anthropic.Model(chatReq.Model),
Messages: messages,
MaxTokens: maxTokens,
Stream: false,
})
if err != nil {
return nil, err
}
c.tokenUsage.PromptTokens += resp.Usage.InputTokens
c.tokenUsage.CompletionTokens += resp.Usage.OutputTokens
c.tokenUsage.TotalTokens += resp.Usage.InputTokens + resp.Usage.OutputTokens
return &llm.ChatResponse{
Model: string(resp.Model),
Choices: []openai.ChatCompletionChoice{
{
FinishReason: openai.FinishReason(resp.StopReason),
Message: openai.ChatCompletionMessage{
Role: openai.ChatMessageRoleAssistant,
Content: *resp.Content[0].Text,
},
},
},
}, nil
}
func (c *Claude) StreamChat(ctx context.Context, chatReq llm.ChatRequest) (chan *llm.StreamChatResponse, error) {
var messages []anthropic.Message
if len(chatReq.Messages) > 0 {
for _, msg := range chatReq.Messages {
var role anthropic.ChatRole
if msg.Role != "assistant" {
role = anthropic.RoleUser
} else {
role = anthropic.RoleAssistant
}
var content []anthropic.MessageContent
if len(msg.MultiContent) > 0 {
for _, mc := range msg.MultiContent {
if mc.Type == "text" {
content = append(content, anthropic.MessageContent{Type: anthropic.MessagesContentTypeText, Text: &mc.Text})
}
if mc.Type == "image_url" {
if strings.HasPrefix(mc.ImageURL.URL, "http") {
continue
}
if strings.HasPrefix(mc.ImageURL.URL, "data:image") {
var mediaType string
if strings.HasPrefix(mc.ImageURL.URL, "data:image/jpeg") {
mediaType = "image/jpeg"
}
if strings.HasPrefix(mc.ImageURL.URL, "data:image/png") {
mediaType = "image/png"
}
imageString := strings.Split(mc.ImageURL.URL, ",")[1]
imageBytes, _ := base64.StdEncoding.DecodeString(imageString)
content = append(content, anthropic.MessageContent{Type: "image", Source: &anthropic.MessageContentSource{Type: "base64", MediaType: mediaType, Data: imageBytes}})
}
}
messages = append(messages, anthropic.Message{Role: role, Content: content})
}
} else {
if len(msg.Content) > 0 {
content = append(content, anthropic.MessageContent{Type: "text", Text: &msg.Content})
}
}
messages = append(messages, anthropic.Message{Role: role, Content: content})
}
}
var maxTokens int
if chatReq.MaxTokens > 0 {
maxTokens = chatReq.MaxTokens
} else {
if strings.Contains(chatReq.Model, "sonnet") || strings.Contains(chatReq.Model, "haiku") {
maxTokens = 8192
} else {
maxTokens = 4096
}
}
datachan := make(chan *llm.StreamChatResponse)
// var resp anthropic.MessagesResponse
var err error
go func() {
defer close(datachan)
_, err = c.Client.CreateMessagesStream(ctx, anthropic.MessagesStreamRequest{
MessagesRequest: anthropic.MessagesRequest{
Model: anthropic.Model(chatReq.Model),
Messages: messages,
MaxTokens: maxTokens,
},
OnContentBlockDelta: func(data anthropic.MessagesEventContentBlockDeltaData) {
datachan <- &llm.StreamChatResponse{
Model: chatReq.Model,
Choices: []openai.ChatCompletionStreamChoice{
{
Delta: openai.ChatCompletionStreamChoiceDelta{Content: *data.Delta.Text},
},
},
}
},
OnMessageStart: func(memss anthropic.MessagesEventMessageStartData) {
c.tokenUsage.PromptTokens += memss.Message.Usage.InputTokens
c.tokenUsage.CompletionTokens += memss.Message.Usage.OutputTokens
c.tokenUsage.TotalTokens += memss.Message.Usage.InputTokens + memss.Message.Usage.OutputTokens
},
OnMessageDelta: func(memdd anthropic.MessagesEventMessageDeltaData) {
c.tokenUsage.PromptTokens += memdd.Usage.InputTokens
c.tokenUsage.CompletionTokens += memdd.Usage.OutputTokens
c.tokenUsage.TotalTokens += memdd.Usage.InputTokens + memdd.Usage.OutputTokens
datachan <- &llm.StreamChatResponse{
Model: chatReq.Model,
Choices: []openai.ChatCompletionStreamChoice{
{FinishReason: openai.FinishReason(memdd.Delta.StopReason)},
},
}
},
})
select {
case <-ctx.Done():
return
default:
}
}()
if err != nil {
return nil, err
}
return datachan, err
}
func (c *Claude) GetTokenUsage() *llm.TokenUsage {
return c.tokenUsage
}