openteam/server/internal/proxy/hold_test.go

package proxy

import (
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/openteam/server/internal/pkg/crypto"
	"github.com/openteam/server/internal/store"
)

// addChannel 给 harness 加一个绑定同一模型的渠道，返回渠道名。
// weight 传大数可以把加权随机的起点压到该渠道上（与既有测试同一手法）。
func addChannel(t *testing.T, h *harness, name, baseURL string, weight int) {
	t.Helper()
	encKey, err := crypto.NewEncryptor("test-master-key-1234567890").Encrypt("upstream-key")
	if err != nil {
		t.Fatal(err)
	}
	var m store.Model
	h.db.Where("name = ?", "m1").First(&m)
	ch := store.Channel{Name: name, Provider: store.ChannelProviderOpenAI,
		Formats: []string{store.FormatChat}, BaseURL: baseURL, APIKeyEnc: encKey,
		Weight: weight, TimeoutMS: 0, MaxConcurrency: 4,
		HealthStatus: store.ChannelHealthHealthy, Enabled: true}
	if err := h.db.Create(&ch).Error; err != nil {
		t.Fatal(err)
	}
	if err := h.db.Create(&store.ChannelModelBinding{ChannelID: ch.ID, ModelID: m.ID, UpstreamModel: "m1"}).Error; err != nil {
		t.Fatal(err)
	}
	h.gw.InvalidateCatalog() // 管理端写操作会失效目录缓存，测试里直接写库要手动失效
}

// pushWeightTo 把指定名字渠道的权重改大，让加权随机几乎必然从它开始。
func pushWeightTo(t *testing.T, h *harness, name string, weight int) {
	t.Helper()
	if err := h.db.Model(&store.Channel{}).Where("name = ?", name).Update("weight", weight).Error; err != nil {
		t.Fatal(err)
	}
	h.gw.InvalidateCatalog()
}

// 流式容错：渠道1 在首个内容事件之前回错误事件（200+流式形式的错误），应换渠道2
// 重试——客户端只看到渠道2 的内容，全程 200，无感知切换。
func TestStreamFailoverBeforeFirstContent(t *testing.T) {
	var ch1Hits, ch2Hits atomic.Int64
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch1Hits.Add(1)
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `data: {"error":{"message":"overloaded","type":"server_error"}}`+"\n\n")
	}))
	defer bad.Close()
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch2Hits.Add(1)
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"来自健康渠道"}}]}`+"\n\n")
		_, _ = io.WriteString(w, "data: [DONE]\n\n")
	}))
	defer good.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	addChannel(t, h, "c2", good.URL, 1)
	pushWeightTo(t, h, "c1", 1000)

	w := h.run(t, nil, `{"model":"m1","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK {
		t.Fatalf("code=%d body=%s", w.Code, w.Body.String())
	}
	out := w.Body.String()
	if ch1Hits.Load() != 1 || ch2Hits.Load() != 1 {
		t.Fatalf("渠道命中次数不符：ch1=%d ch2=%d", ch1Hits.Load(), ch2Hits.Load())
	}
	if strings.Contains(out, "overloaded") {
		t.Fatalf("渠道1 的错误不应到达客户端：%q", out)
	}
	if !strings.Contains(out, "来自健康渠道") {
		t.Fatalf("应收到渠道2 的内容：%q", out)
	}
}

// Anthropic 形状的错误事件（{"type":"error","error":{...}}）同样在放行前被识别换渠道。
func TestAnthropicStyleStreamErrorFailsOver(t *testing.T) {
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `event: error`+"\n"+`data: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`+"\n\n")
	}))
	defer bad.Close()
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"ok"}}]}`+"\n\n")
		_, _ = io.WriteString(w, "data: [DONE]\n\n")
	}))
	defer good.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	addChannel(t, h, "c2", good.URL, 1)
	pushWeightTo(t, h, "c1", 1000)

	w := h.run(t, nil, `{"model":"m1","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"content":"ok"`) {
		t.Fatalf("应无感知切换到渠道2：code=%d body=%s", w.Code, w.Body.String())
	}
}

// 渠道回了 200+event-stream 却一个字没写就断流：放行前视为失败，换渠道重试。
func TestEmptyStreamBeforeContentFailsOver(t *testing.T) {
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		w.(http.Flusher).Flush()
		// 直接结束：EOF 且无内容
	}))
	defer bad.Close()
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"补位渠道"}}]}`+"\n\n")
		_, _ = io.WriteString(w, "data: [DONE]\n\n")
	}))
	defer good.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	addChannel(t, h, "c2", good.URL, 1)
	pushWeightTo(t, h, "c1", 1000)

	w := h.run(t, nil, `{"model":"m1","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "补位渠道") {
		t.Fatalf("空流应触发故障转移：code=%d body=%s", w.Code, w.Body.String())
	}
}

// didNotStream：要流式，渠道却回了整包 JSON——视为失败换渠道，而不是把 JSON 塞给等 SSE 的客户端。
func TestDidNotStreamFailsOver(t *testing.T) {
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `{"id":"x","choices":[{"message":{"role":"assistant","content":"整包"}}]}`)
	}))
	defer bad.Close()
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"真流式"}}]}`+"\n\n")
		_, _ = io.WriteString(w, "data: [DONE]\n\n")
	}))
	defer good.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	addChannel(t, h, "c2", good.URL, 1)
	pushWeightTo(t, h, "c1", 1000)

	w := h.run(t, nil, `{"model":"m1","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK {
		t.Fatalf("code=%d body=%s", w.Code, w.Body.String())
	}
	out := w.Body.String()
	if !strings.HasPrefix(out, "data:") || !strings.Contains(out, "真流式") {
		t.Fatalf("应从渠道2 收到 SSE 流：%q", out)
	}
}

// 反向 didNotStream：要整包，渠道回了 event-stream，同样换渠道。
func TestNonStreamReceivingStreamFailsOver(t *testing.T) {
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, "data: [DONE]\n\n")
	}))
	defer bad.Close()
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		_, _ = io.WriteString(w, `{"id":"ok","choices":[{"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
	}))
	defer good.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	addChannel(t, h, "c2", good.URL, 1)
	pushWeightTo(t, h, "c1", 1000)

	w := h.run(t, nil, `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"content":"hi"`) {
		t.Fatalf("应从渠道2 收到整包 JSON：code=%d body=%s", w.Code, w.Body.String())
	}
}

// 首 token 计时：流式放行前迟迟不出内容，超时取消本次尝试、换下一个渠道。
func TestFirstTokenTimeoutFailsOver(t *testing.T) {
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		w.(http.Flusher).Flush()
		select {
		case <-r.Context().Done(): // 被首 token 计时器取消
		case <-time.After(5 * time.Second):
		}
	}))
	defer bad.Close()
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"及时的内容"}}]}`+"\n\n")
		_, _ = io.WriteString(w, "data: [DONE]\n\n")
	}))
	defer good.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	h.gw.firstTokenWait = 150 * time.Millisecond
	addChannel(t, h, "c2", good.URL, 1)
	pushWeightTo(t, h, "c1", 1000)

	w := h.run(t, nil, `{"model":"m1","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "及时的内容") {
		t.Fatalf("首 token 超时应换渠道：code=%d body=%s", w.Code, w.Body.String())
	}
}

// 失败尝试扫到的 usage 不得计入最终账单：渠道1 的 message_start 式用量随失败被丢弃。
func TestUsageFromFailedTryNotBilled(t *testing.T) {
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		// role delta + usage 但没有内容：usage 会被扫进 trySink，随后错误事件导致整次尝试废弃
		_, _ = io.WriteString(w, `data: {"choices":[{"delta":{"role":"assistant"}}],"usage":{"prompt_tokens":500,"completion_tokens":0}}`+"\n\n")
		_, _ = io.WriteString(w, `data: {"error":{"message":"boom"}}`+"\n\n")
	}))
	defer bad.Close()
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, `data: {"choices":[{"delta":{"content":"答案"}}]}`+"\n\n")
		_, _ = io.WriteString(w, `data: {"choices":[],"usage":{"prompt_tokens":7,"completion_tokens":3}}`+"\n\n")
		_, _ = io.WriteString(w, "data: [DONE]\n\n")
	}))
	defer good.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	addChannel(t, h, "c2", good.URL, 1)
	pushWeightTo(t, h, "c1", 1000)

	w := h.run(t, nil, `{"model":"m1","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK {
		t.Fatalf("code=%d body=%s", w.Code, w.Body.String())
	}
	h.rec.Close() // usage 异步批量落库，先冲刷再查
	var logs []store.UsageLog
	h.db.Where("status = ?", store.UsageStatusSuccess).Find(&logs)
	if len(logs) != 1 {
		t.Fatalf("成功记账应只有一条，实际 %d", len(logs))
	}
	if logs[0].InputTokens != 7 || logs[0].OutputTokens != 3 {
		t.Fatalf("应只计渠道2 的用量 (7/3)，实际 in=%d out=%d", logs[0].InputTokens, logs[0].OutputTokens)
	}
}

// 响应头已被心跳先行写出后，全部候选失败只能以 SSE 错误事件收尾（状态码已不可改）。
func TestStreamErrorEventWhenHeadAlreadySent(t *testing.T) {
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.WriteHeader(http.StatusOK)
		time.Sleep(200 * time.Millisecond) // 期间心跳先把 200 头写给客户端
		_, _ = io.WriteString(w, `data: {"error":{"message":"boom"}}`+"\n\n")
	}))
	defer bad.Close()

	h := newHarness(t, bad.URL, 0, true, nil)
	h.gw.streamHeartbeat = 40 * time.Millisecond

	w := h.run(t, nil, `{"model":"m1","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
	if w.Code != http.StatusOK {
		t.Fatalf("头已写出，状态应为 200：code=%d body=%s", w.Code, w.Body.String())
	}
	out := w.Body.String()
	if !strings.Contains(out, ": ping") {
		t.Fatalf("扣住期间应有心跳：%q", out)
	}
	if !strings.Contains(out, `"upstream_error"`) {
		t.Fatalf("应以 SSE 错误事件收尾：%q", out)
	}
}

// ---------------------------------------------------------------------------
// 判定函数单测

func TestSSEEventHasContent(t *testing.T) {
	cases := []struct {
		proto string
		line  string
		want  bool
	}{
		// chat：role-only 与空 delta 不算内容，content/reasoning/tool_calls 算
		{"chat", `data: {"choices":[{"delta":{"role":"assistant"}}]}`, false},
		{"chat", `data: {"choices":[{"delta":{}}]}`, false},
		{"chat", `data: {"choices":[{"delta":{"content":"x"}}]}`, true},
		{"chat", `data: {"choices":[{"delta":{"reasoning_content":"想"}}]}`, true},
		{"chat", `data: {"choices":[{"delta":{"tool_calls":[{"id":"1"}]}}]}`, true},
		{"chat", "data: [DONE]", false},
		// messages：message_start / ping 不算，块开启与 delta 算
		{"messages", `data: {"type":"message_start","message":{"usage":{"input_tokens":1}}}`, false},
		{"messages", `data: {"type":"ping"}`, false},
		{"messages", `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, true},
		{"messages", `data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"t","name":"f"}}`, true},
		{"messages", `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"x"}}`, true},
		{"messages", `data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"想"}}`, true},
		// responses：.delta 与 output_item.added 算，created/completed 不算
		{"responses", `data: {"type":"response.created","response":{}}`, false},
		{"responses", `data: {"type":"response.output_text.delta","delta":"x"}`, true},
		{"responses", `data: {"type":"response.output_item.added","item":{"type":"function_call"}}`, true},
	}
	for i, tc := range cases {
		if got := sseEventHasContent([]byte(tc.line), tc.proto); got != tc.want {
			t.Fatalf("case %d (%s): sseEventHasContent(%s) = %v, want %v", i, tc.proto, tc.line, got, tc.want)
		}
	}
	// 未知协议：任何数据事件都算内容（退化放行，不吞正文）
	if !sseEventHasContent([]byte(`data: {"weird":true}`), "other") {
		t.Fatal("未知协议应放行")
	}
}

func TestSSEUpstreamError(t *testing.T) {
	cases := []struct {
		line    string
		want    bool
		contain string
	}{
		{`data: {"error":{"message":"overloaded","type":"server_error"}}`, true, "overloaded"},
		{`data: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`, true, "Overloaded"},
		{`data: {"type":"error","code":500}`, true, "upstream stream error"},
		{`data: {"type":"response.failed","response":{"error":{"message":"bad tool"}}}`, true, "bad tool"},
		{`data: {"choices":[{"delta":{"content":"hi"}}]}`, false, ""},
		{`data: {"type":"content_block_start","content_block":{"type":"text"}}`, false, ""},
		{`data: {"error":null}`, false, ""},
		{`event: message_start`, false, ""},
	}
	for i, tc := range cases {
		msg, ok := sseUpstreamError([]byte(tc.line))
		if ok != tc.want {
			t.Fatalf("case %d: sseUpstreamError(%s) ok=%v, want %v", i, tc.line, ok, tc.want)
		}
		if ok && !strings.Contains(msg, tc.contain) {
			t.Fatalf("case %d: msg=%q, want contains %q", i, msg, tc.contain)
		}
	}
}

func TestClassifyUpstreamError(t *testing.T) {
	cases := []struct {
		status    int
		body      string
		kind      string
		retryable bool
	}{
		{http.StatusTooManyRequests, `{"error":{"message":"Rate limit reached"}}`, "rate_limited", true},
		{http.StatusTooManyRequests, `{"error":{"message":"You exceeded your current quota, check your plan and billing details"}}`, "quota_exhausted", true},
		{http.StatusTooManyRequests, `{"error":{"code":"1113","message":"余额不足"}}`, "quota_exhausted", true},
		{http.StatusPaymentRequired, `{"error":{"message":"insufficient balance"}}`, "quota_exhausted", true},
		{http.StatusForbidden, `{"error":{"message":"insufficient quota"}}`, "quota_exhausted", true},
		{http.StatusForbidden, `{"error":{"message":"model not allowed"}}`, "server_error", false},
		{http.StatusBadGateway, `{"error":{"message":"upstream gone"}}`, "server_error", true},
		{http.StatusBadRequest, `{"error":{"message":"bad request"}}`, "server_error", false},
	}
	for i, tc := range cases {
		kind, retryable := classifyUpstreamError(tc.status, []byte(tc.body))
		if kind.String() != tc.kind || retryable != tc.retryable {
			t.Fatalf("case %d: classify(%d) = (%s, %v), want (%s, %v)", i, tc.status, kind, retryable, tc.kind, tc.retryable)
		}
	}
}

func TestStreamErrorEventShapes(t *testing.T) {
	chat := string(streamErrorEvent("chat", 502, "boom"))
	if !strings.HasPrefix(chat, "data: ") || !strings.Contains(chat, `"code":"upstream_error"`) || !strings.Contains(chat, "data: [DONE]") {
		t.Fatalf("chat 错误事件形状不符：%q", chat)
	}
	msg := string(streamErrorEvent("messages", 502, "boom"))
	if !strings.HasPrefix(msg, "event: error\ndata: ") {
		t.Fatalf("messages 错误事件形状不符：%q", msg)
	}
	var m map[string]any
	if err := json.Unmarshal([]byte(strings.TrimPrefix(strings.SplitN(msg, "data: ", 2)[1], "\n\n")), &m); err != nil {
		t.Fatalf("messages 错误事件不是合法 JSON: %v (%s)", err, msg)
	}
	resp := string(streamErrorEvent("responses", 502, "boom"))
	if !strings.HasPrefix(resp, "data: ") || strings.Contains(resp, "[DONE]") {
		t.Fatalf("responses 错误事件形状不符：%q", resp)
	}
}