diff --git a/server/internal/proxy/convert/convert_test.go b/server/internal/proxy/convert/convert_test.go index 9e0c9cf..b38ba81 100644 --- a/server/internal/proxy/convert/convert_test.go +++ b/server/internal/proxy/convert/convert_test.go @@ -306,3 +306,246 @@ func TestStreamMessagesToResponses(t *testing.T) { t.Fatalf("missing response.completed: %s", out) } } + +// --------------------------------------------------------------------------- +// 图片内容块转换(messages image / chat image_url / responses input_image 互转不丢图) + +const testB64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + +// userContentBlocks 断言 Chat 用户消息 content 为块数组并返回。 +func userContentBlocks(t *testing.T, out []byte) []any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(out, &m); err != nil { + t.Fatalf("unmarshal: %v\n%s", err, out) + } + msgs := m["messages"].([]any) + if len(msgs) != 1 { + t.Fatalf("messages len = %d: %s", len(msgs), out) + } + content, ok := msgs[0].(map[string]any)["content"].([]any) + if !ok { + t.Fatalf("content not array: %s", out) + } + return content +} + +// blockURL 取 image_url 块的 url。 +func blockURL(t *testing.T, b any) string { + t.Helper() + bm := b.(map[string]any) + if bm["type"] != "image_url" { + t.Fatalf("block type = %v", bm["type"]) + } + iu := bm["image_url"].(map[string]any) + url, _ := iu["url"].(string) + return url +} + +func TestMessagesToChatReqImage(t *testing.T) { + in := mustJSON(t, map[string]any{ + "model": "gpt-4o-mini", + "messages": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "text", "text": "describe"}, + map[string]any{"type": "image", "source": map[string]any{"type": "base64", "media_type": "image/png", "data": testB64}}, + map[string]any{"type": "image", "source": map[string]any{"type": "url", "url": "https://example.com/a.png"}}, + }}}, + }) + out, err := ConvertRequest([]byte(in), ProtoMessages, ProtoChat) + if err != nil { + t.Fatalf("convert: %v", err) + } + blocks := userContentBlocks(t, out) + if len(blocks) != 3 { + t.Fatalf("blocks len = %d: %s", len(blocks), out) + } + if got := blockURL(t, blocks[1]); got != "data:image/png;base64,"+testB64 { + t.Fatalf("base64 image url = %q", got) + } + if got := blockURL(t, blocks[2]); got != "https://example.com/a.png" { + t.Fatalf("url image = %q", got) + } +} + +// 多图 + 图/文交错顺序:转换后块数量与顺序保持不变(文档推荐的 Image 1:/Image 2: 标签模式)。 +func TestMessagesToChatReqMultiImage(t *testing.T) { + img := func(url string) any { + return map[string]any{"type": "image", "source": map[string]any{"type": "url", "url": url}} + } + in := mustJSON(t, map[string]any{ + "model": "gpt-4o-mini", + "messages": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "text", "text": "Image 1:"}, + img("https://example.com/1.png"), + map[string]any{"type": "text", "text": "Image 2:"}, + img("https://example.com/2.png"), + map[string]any{"type": "text", "text": "How are these images different?"}, + }}}, + }) + out, err := ConvertRequest([]byte(in), ProtoMessages, ProtoChat) + if err != nil { + t.Fatalf("convert: %v", err) + } + blocks := userContentBlocks(t, out) + if len(blocks) != 5 { + t.Fatalf("blocks len = %d: %s", len(blocks), out) + } + types := []string{} + for _, b := range blocks { + types = append(types, b.(map[string]any)["type"].(string)) + } + want := []string{"text", "image_url", "text", "image_url", "text"} + for i := range want { + if types[i] != want[i] { + t.Fatalf("block[%d] type = %s, want %s: %v", i, types[i], want[i], types) + } + } + // 第二张图仍是第二张 + if got := blockURL(t, blocks[3]); got != "https://example.com/2.png" { + t.Fatalf("image2 url = %q", got) + } +} + +func TestChatToMessagesReqImage(t *testing.T) { + in := mustJSON(t, map[string]any{ + "model": "claude-sonnet-5", + "messages": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "text", "text": "describe"}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/jpeg;base64," + testB64}}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.com/b.png"}}, + }}}, + }) + out, err := ConvertRequest([]byte(in), ProtoChat, ProtoMessages) + if err != nil { + t.Fatalf("convert: %v", err) + } + var m map[string]any + _ = json.Unmarshal(out, &m) + msgs := m["messages"].([]any) + content := msgs[0].(map[string]any)["content"].([]any) + if len(content) != 3 { + t.Fatalf("content len = %d: %s", len(content), out) + } + src1 := content[1].(map[string]any)["source"].(map[string]any) + if src1["type"] != "base64" || src1["media_type"] != "image/jpeg" || src1["data"] != testB64 { + t.Fatalf("base64 source = %v", src1) + } + src2 := content[2].(map[string]any)["source"].(map[string]any) + if src2["type"] != "url" || src2["url"] != "https://example.com/b.png" { + t.Fatalf("url source = %v", src2) + } +} + +func TestResponsesToChatReqImage(t *testing.T) { + in := mustJSON(t, map[string]any{ + "model": "claude-sonnet-5", + "input": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "input_text", "text": "describe"}, + map[string]any{"type": "input_image", "image_url": "data:image/png;base64," + testB64}, + }}}, + }) + out, err := ConvertRequest([]byte(in), ProtoResponses, ProtoChat) + if err != nil { + t.Fatalf("convert: %v", err) + } + blocks := userContentBlocks(t, out) + if len(blocks) != 2 { + t.Fatalf("blocks len = %d: %s", len(blocks), out) + } + if got := blockURL(t, blocks[1]); got != "data:image/png;base64,"+testB64 { + t.Fatalf("image url = %q", got) + } +} + +func TestChatToResponsesReqImage(t *testing.T) { + in := mustJSON(t, map[string]any{ + "model": "gpt-4o", + "messages": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "text", "text": "describe"}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.com/c.png"}}, + }}}, + }) + out, err := ConvertRequest([]byte(in), ProtoChat, ProtoResponses) + if err != nil { + t.Fatalf("convert: %v", err) + } + var m map[string]any + _ = json.Unmarshal(out, &m) + var input []map[string]any + // 单条 user 消息项 + switch v := m["input"].(type) { + case map[string]any: + input = []map[string]any{v} + case []any: + for _, it := range v { + input = append(input, it.(map[string]any)) + } + } + content := input[0]["content"].([]any) + if len(content) != 2 { + t.Fatalf("content len = %d: %s", len(content), out) + } + img := content[1].(map[string]any) + if img["type"] != "input_image" { + t.Fatalf("block type = %v", img["type"]) + } + if img["image_url"] != "https://example.com/c.png" { + t.Fatalf("image_url = %v", img["image_url"]) + } +} + +// 链式转换:responses→messages、messages→responses(经 chat 中转)也不丢图。 +func TestResponsesToMessagesReqImage(t *testing.T) { + in := mustJSON(t, map[string]any{ + "model": "claude-sonnet-5", + "input": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "input_text", "text": "describe"}, + map[string]any{"type": "input_image", "image_url": "data:image/png;base64," + testB64}, + }}}, + }) + out, err := ConvertRequest([]byte(in), ProtoResponses, ProtoMessages) + if err != nil { + t.Fatalf("convert: %v", err) + } + var m map[string]any + _ = json.Unmarshal(out, &m) + msgs := m["messages"].([]any) + content := msgs[0].(map[string]any)["content"].([]any) + if len(content) != 2 { + t.Fatalf("content len = %d: %s", len(content), out) + } + src := content[1].(map[string]any)["source"].(map[string]any) + if src["type"] != "base64" || src["data"] != testB64 { + t.Fatalf("source = %v", src) + } +} + +func TestMessagesToResponsesReqImage(t *testing.T) { + in := mustJSON(t, map[string]any{ + "model": "gpt-4o", + "messages": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "image", "source": map[string]any{"type": "url", "url": "https://example.com/d.png"}}, + map[string]any{"type": "text", "text": "describe"}, + }}}, + }) + out, err := ConvertRequest([]byte(in), ProtoMessages, ProtoResponses) + if err != nil { + t.Fatalf("convert: %v", err) + } + var m map[string]any + _ = json.Unmarshal(out, &m) + var content []any + switch v := m["input"].(type) { + case map[string]any: + content = v["content"].([]any) + case []any: + content = v[0].(map[string]any)["content"].([]any) + } + if len(content) != 2 { + t.Fatalf("content len = %d: %s", len(content), out) + } + img := content[0].(map[string]any) + if img["type"] != "input_image" || img["image_url"] != "https://example.com/d.png" { + t.Fatalf("input_image = %v", img) + } +} diff --git a/server/internal/proxy/convert/json_chat.go b/server/internal/proxy/convert/json_chat.go index 27042f3..c1a0a1d 100644 --- a/server/internal/proxy/convert/json_chat.go +++ b/server/internal/proxy/convert/json_chat.go @@ -137,9 +137,11 @@ func chatMsgToAnthropic(m chatMsg) any { var url string if iu, ok := b["image_url"].(map[string]any); ok { url, _ = iu["url"].(string) + } else if s, ok := b["image_url"].(string); ok { + url = s } if url != "" { - blocks = append(blocks, map[string]any{"type": "image", "source": map[string]any{"type": "url", "url": url}}) + blocks = append(blocks, anthropicImageBlock(url)) } } } @@ -229,12 +231,18 @@ func anthropicMsgToChat(role string, content json.RawMessage) []any { if json.Unmarshal(content, &blocks) == nil && blocks != nil { var out []any var textParts []string + var contentBlocks []any // text / image_url 块,保留原始顺序 var toolCalls []any for _, b := range blocks { switch b["type"] { case "text": if t, _ := b["text"].(string); t != "" { textParts = append(textParts, t) + contentBlocks = append(contentBlocks, map[string]any{"type": "text", "text": t}) + } + case "image": + if cb := chatImageBlock(b); cb != nil { + contentBlocks = append(contentBlocks, cb) } case "tool_use": id, _ := b["id"].(string) @@ -254,9 +262,19 @@ func anthropicMsgToChat(role string, content json.RawMessage) []any { out = append(out, map[string]any{"role": "tool", "tool_call_id": callID, "content": res}) } } - if len(textParts) > 0 || len(toolCalls) > 0 { + hasImage := false + for _, cb := range contentBlocks { + if m, _ := cb.(map[string]any); m["type"] == "image_url" { + hasImage = true + break + } + } + if hasImage || len(textParts) > 0 || len(toolCalls) > 0 { msg := map[string]any{"role": role} - if len(textParts) > 0 { + switch { + case hasImage: + msg["content"] = contentBlocks + case len(textParts) > 0: msg["content"] = strings.Join(textParts, "") } if len(toolCalls) > 0 { @@ -416,6 +434,58 @@ func intOrNil(p *int, def int) any { return *p } +// splitDataURL 解析 data:media_type;base64,data 形式的 URL;非该形式返回 ok=false。 +func splitDataURL(url string) (media, data string, ok bool) { + if !strings.HasPrefix(url, "data:") { + return "", "", false + } + i := strings.Index(url, ";base64,") + if i < 0 { + return "", "", false + } + return url[len("data:"):i], url[i+len(";base64,"):], true +} + +// chatImageBlock 把 Anthropic image 块转 OpenAI image_url 块。 +// 仅支持 base64 与 url source;其他类型(如 Files API 的 file_id)不支持,跳过。 +func chatImageBlock(b map[string]any) any { + src, ok := b["source"].(map[string]any) + if !ok { + return nil + } + switch src["type"] { + case "base64": + media, _ := src["media_type"].(string) + data, _ := src["data"].(string) + if data == "" { + return nil + } + if media == "" { + media = "image/png" + } + return map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:" + media + ";base64," + data}} + case "url": + url, _ := src["url"].(string) + if url == "" { + return nil + } + return map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}} + } + return nil +} + +// anthropicImageBlock 把 OpenAI image_url 的 url 转 Anthropic image 块。 +// data URL → base64 source;http(s) URL → url source。 +func anthropicImageBlock(url string) any { + if media, data, ok := splitDataURL(url); ok { + if media == "" { + media = "image/png" + } + return map[string]any{"type": "image", "source": map[string]any{"type": "base64", "media_type": media, "data": data}} + } + return map[string]any{"type": "image", "source": map[string]any{"type": "url", "url": url}} +} + func rawOrObject(raw json.RawMessage) any { if len(raw) == 0 || string(raw) == "null" { return map[string]any{} diff --git a/server/internal/proxy/convert/json_responses.go b/server/internal/proxy/convert/json_responses.go index 7c70a19..2524fd5 100644 --- a/server/internal/proxy/convert/json_responses.go +++ b/server/internal/proxy/convert/json_responses.go @@ -101,20 +101,82 @@ func responsesInputToChat(raw json.RawMessage) []any { out = append(out, map[string]any{"role": role, "content": content}) } else if blocks, ok := item["content"].([]any); ok { var text []string + var contentBlocks []any for _, b := range blocks { - if bm, ok := b.(map[string]any); ok { + bm, ok := b.(map[string]any) + if !ok { + continue + } + switch bm["type"] { + case "input_text", "text": if t, _ := bm["text"].(string); t != "" { text = append(text, t) + contentBlocks = append(contentBlocks, map[string]any{"type": "text", "text": t}) + } + case "input_image": + var url string + if s, ok := bm["image_url"].(string); ok { + url = s + } else if m, ok := bm["image_url"].(map[string]any); ok { + url, _ = m["url"].(string) + } + if url != "" { + contentBlocks = append(contentBlocks, map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}}) } } } - out = append(out, map[string]any{"role": role, "content": strings.Join(text, "")}) + hasImage := false + for _, cb := range contentBlocks { + if m, _ := cb.(map[string]any); m["type"] == "image_url" { + hasImage = true + break + } + } + if hasImage { + out = append(out, map[string]any{"role": role, "content": contentBlocks}) + } else { + out = append(out, map[string]any{"role": role, "content": strings.Join(text, "")}) + } } } } return out } +// chatContentToResponsesBlocks 把 Chat 用户消息 content 转 Responses input 块数组(input_text / input_image)。 +func chatContentToResponsesBlocks(content json.RawMessage) []any { + // 纯字符串 → 单个 input_text + var s string + if json.Unmarshal(content, &s) == nil && s != "" { + return []any{map[string]any{"type": "input_text", "text": s}} + } + // 数组 → 按块转换(text / image_url) + var arr []map[string]any + if json.Unmarshal(content, &arr) == nil && arr != nil { + var out []any + for _, b := range arr { + switch b["type"] { + case "text", "input_text": + if t, _ := b["text"].(string); t != "" { + out = append(out, map[string]any{"type": "input_text", "text": t}) + } + case "image_url": + var url string + if iu, ok := b["image_url"].(map[string]any); ok { + url, _ = iu["url"].(string) + } else if s, ok := b["image_url"].(string); ok { + url = s + } + if url != "" { + out = append(out, map[string]any{"type": "input_image", "image_url": url}) + } + } + } + return out + } + return nil +} + // --------------------------------------------------------------------------- // 请求:Chat → Responses @@ -170,10 +232,8 @@ func chatToResponsesReq(body []byte) ([]byte, error) { }}) } default: - if s := str(m.Content); s != "" { - input = append(input, map[string]any{"type": "message", "role": "user", "content": []any{ - map[string]any{"type": "input_text", "text": s}, - }}) + if blocks := chatContentToResponsesBlocks(m.Content); len(blocks) > 0 { + input = append(input, map[string]any{"type": "message", "role": "user", "content": blocks}) } } }