update: openai struct

pkg/claude/chat.go

@@ -143,51 +143,57 @@ func ChatMessages(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
 		claudReq.Model = ""
 	}
 
-	var prompt string
-
 	var claudecontent []VisionContent
+	var prompt string
 	for _, msg := range chatReq.Messages {
-		if msg.Role == "system" {
+		switch ct := msg.Content.(type) {
-			claudReq.System = string(msg.Content)
+		case string:
-			continue
+			prompt += "<" + msg.Role + ">: " + msg.Content.(string) + "\n"
-		}
+			if msg.Role == "system" {
-
+				claudReq.System = msg.Content.(string)
-		var oaivisioncontent []openai.VisionContent
+				continue
-		if err := json.Unmarshal(msg.Content, &oaivisioncontent); err != nil {
+			}
-			prompt += "<" + msg.Role + ">: " + string(msg.Content) + "\n"
+			claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + msg.Content.(string)})
-
+		case []any:
-			claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + string(msg.Content)})
+			for _, item := range ct {
-		} else {
+				if m, ok := item.(map[string]interface{}); ok {
-			if len(oaivisioncontent) > 0 {
+					if m["type"] == "text" {
-				for _, content := range oaivisioncontent {
+						prompt += "<" + msg.Role + ">: " + m["text"].(string) + "\n"
-					if content.Type == "text" {
+						claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + m["text"].(string)})
-						prompt += "<" + msg.Role + ">: " + content.Text + "\n"
+					} else if m["type"] == "image_url" {
-						claudecontent = append(claudecontent, VisionContent{Type: "text", Text: msg.Role + ":" + content.Text})
+						if url, ok := m["image_url"].(map[string]interface{}); ok {
-					} else if content.Type == "image_url" {
+							fmt.Printf("  URL: %v\n", url["url"])
-						if strings.HasPrefix(content.ImageURL.URL, "http") {
+							if strings.HasPrefix(url["url"].(string), "http") {
-							fmt.Println("链接:", content.ImageURL.URL)
+								fmt.Println("网络图片:", url["url"].(string))
-						} else if strings.HasPrefix(content.ImageURL.URL, "data:image") {
+							} else if strings.HasPrefix(url["url"].(string), "data:image") {
-							fmt.Println("base64:", content.ImageURL.URL[:20])
+								fmt.Println("base64:", url["url"].(string)[:20])
+								var mediaType string
+								if strings.HasPrefix(url["url"].(string), "data:image/jpeg") {
+									mediaType = "image/jpeg"
+								}
+								if strings.HasPrefix(url["url"].(string), "data:image/png") {
+									mediaType = "image/png"
+								}
+								claudecontent = append(claudecontent, VisionContent{Type: "image", Source: &VisionSource{Type: "base64", MediaType: mediaType, Data: strings.Split(url["url"].(string), ",")[1]}})
+							}
 						}
-						// todo image tokens
-						var mediaType string
-						if strings.HasPrefix(content.ImageURL.URL, "data:image/jpeg") {
-							mediaType = "image/jpeg"
-						}
-						if strings.HasPrefix(content.ImageURL.URL, "data:image/png") {
-							mediaType = "image/png"
-						}
-						claudecontent = append(claudecontent, VisionContent{Type: "image", Source: &VisionSource{Type: "base64", MediaType: mediaType, Data: strings.Split(content.ImageURL.URL, ",")[1]}})
 					}
 				}
-
 			}
+		default:
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error": gin.H{
+					"message": "Invalid content type",
+				},
+			})
+			return
+		}
+		if len(chatReq.Tools) > 0 {
+			tooljson, _ := json.Marshal(chatReq.Tools)
+			prompt += "<tools>: " + string(tooljson) + "\n"
 		}
-		// if len(chatReq.Tools) > 0 {
-		// 	tooljson, _ := json.Marshal(chatReq.Tools)
-		// 	prompt += "<tools>: " + string(tooljson) + "\n"
-		// }
 	}
+
 	claudReq.Messages = []VisionMessages{{Role: "user", Content: claudecontent}}
 
 	usagelog.PromptCount = tokenizer.NumTokensFromStr(prompt, chatReq.Model)

pkg/google/chat.go

@@ -97,59 +97,55 @@ func ChatProxy(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
 	var prompts []genai.Part
 	var prompt string
 	for _, msg := range chatReq.Messages {
-		var visioncontent []openai.VisionContent
+		switch ct := msg.Content.(type) {
-		if err := json.Unmarshal(msg.Content, &visioncontent); err != nil {
+		case string:
-			prompt += "<" + msg.Role + ">: " + string(msg.Content) + "\n"
+			prompt += "<" + msg.Role + ">: " + msg.Content.(string) + "\n"
-			prompts = append(prompts, genai.Text("<"+msg.Role+">: "+string(msg.Content)))
+			prompts = append(prompts, genai.Text("<"+msg.Role+">: "+msg.Content.(string)))
-		} else {
+		case []any:
-			if len(visioncontent) > 0 {
+			for _, item := range ct {
-				for _, content := range visioncontent {
+				if m, ok := item.(map[string]interface{}); ok {
-					if content.Type == "text" {
+					if m["type"] == "text" {
-						prompt += "<" + msg.Role + ">: " + content.Text + "\n"
+						prompt += "<" + msg.Role + ">: " + m["text"].(string) + "\n"
-						prompts = append(prompts, genai.Text("<"+msg.Role+">: "+content.Text))
+						prompts = append(prompts, genai.Text("<"+msg.Role+">: "+m["text"].(string)))
-					} else if content.Type == "image_url" {
+					} else if m["type"] == "image_url" {
-						if strings.HasPrefix(content.ImageURL.URL, "http") {
+						if url, ok := m["image_url"].(map[string]interface{}); ok {
-							fmt.Println("链接:", content.ImageURL.URL)
+							if strings.HasPrefix(url["url"].(string), "http") {
-						} else if strings.HasPrefix(content.ImageURL.URL, "data:image") {
+								fmt.Println("网络图片:", url["url"].(string))
-							fmt.Println("base64:", content.ImageURL.URL[:20])
+							} else if strings.HasPrefix(url["url"].(string), "data:image") {
-							if chatReq.Model != "gemini-pro-vision" {
+								fmt.Println("base64:", url["url"].(string)[:20])
-								chatReq.Model = "gemini-pro-vision"
+								var mime string
+								// openai 会以 data:image 开头，则去掉 data:image/png;base64, 和 data:image/jpeg;base64,
+								if strings.HasPrefix(url["url"].(string), "data:image/png") {
+									mime = "image/png"
+								} else if strings.HasPrefix(url["url"].(string), "data:image/jpeg") {
+									mime = "image/jpeg"
+								} else {
+									c.JSON(http.StatusInternalServerError, gin.H{"error": "Unsupported image format"})
+									return
+								}
+								imageString := strings.Split(url["url"].(string), ",")[1]
+								imageBytes, err := base64.StdEncoding.DecodeString(imageString)
+								if err != nil {
+									c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+									return
+								}
+								prompts = append(prompts, genai.Blob{MIMEType: mime, Data: imageBytes})
 							}
-
-							var mime string
-							// openai 会以 data:image 开头，则去掉 data:image/png;base64, 和 data:image/jpeg;base64,
-							if strings.HasPrefix(content.ImageURL.URL, "data:image/png") {
-								mime = "image/png"
-							} else if strings.HasPrefix(content.ImageURL.URL, "data:image/jpeg") {
-								mime = "image/jpeg"
-							} else {
-								c.JSON(http.StatusInternalServerError, gin.H{"error": "Unsupported image format"})
-								return
-							}
-							imageString := strings.Split(content.ImageURL.URL, ",")[1]
-							imageBytes, err := base64.StdEncoding.DecodeString(imageString)
-							if err != nil {
-								c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-								return
-							}
-							prompts = append(prompts, genai.Blob{MIMEType: mime, Data: imageBytes})
 						}
-
-						// todo image tokens
 					}
-
 				}
-
 			}
+		default:
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error": gin.H{
+					"message": "Invalid content type",
+				},
+			})
+			return
 		}
 		if len(chatReq.Tools) > 0 {
 			tooljson, _ := json.Marshal(chatReq.Tools)
 			prompt += "<tools>: " + string(tooljson) + "\n"
-
-			// for _, tool := range chatReq.Tools {
-
-			// }
-
 		}
 	}
 
@@ -171,6 +167,7 @@ func ChatProxy(c *gin.Context, chatReq *openai.ChatCompletionRequest) {
 	defer client.Close()
 
 	model := client.GenerativeModel(chatReq.Model)
+	model.Tools = []*genai.Tool{}
 
 	iter := model.GenerateContentStream(ctx, prompts...)
 	datachan := make(chan string)