M5: 渠道体系(负载均衡+并发控制+健康检查+故障转移)

- channel.Candidates 按模型绑定取候选 + Pick 加权随机负载均衡
- TryAcquire 每渠道并发信号量, 满载溢出到其他候选
- HealthMonitor 后台定时探测, 连续失败进 cooldown, 恢复放回(可配 interval/threshold)
- doProxy 遍历候选故障转移: 网络错误/429/5xx/超时且未写出响应头时安全重试;
  400 等业务错误透传, 流式写出首字节后放弃重试
- 单测覆盖候选过滤/加权/并发/健康状态机
- E2E: 杀上游自动切换、cooldown、恢复、并发溢出 10/10

Co-Authored-By: Claude <noreply@anthropic.com>

server/internal/proxy/gateway.go

@@ -123,14 +123,9 @@ func (g *Gateway) models(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{"object": "list", "data": data})
 }
 
-// selectChannel 选渠道：优先按模型绑定解析，退化为全局选渠道。
-func (g *Gateway) selectChannel(c *gin.Context, model string) (*store.Channel, error) {
-	if model != "" {
-		if ch, _, err := g.ch.ResolveModel(model); err == nil {
-			return ch, nil
-		}
-	}
-	return g.ch.Select()
+// candidateChannels 返回可用渠道候选（按模型绑定优先，退化全局）。
+func (g *Gateway) candidateChannels(model string) []*store.Channel {
+	return g.ch.Candidates(model)
 }
 
 // resolveUser 取当前用户（含余额）。

server/internal/proxy/handlers.go

@@ -24,20 +24,15 @@ func (g *Gateway) chatCompletions(c *gin.Context) {
 	c.Set("protocol", convert.ProtoChat)
 	c.Set("model_name", br.Model)
 
-	ch, err := g.selectChannel(c, br.Model)
-	if err != nil {
+	cands := g.candidateChannels(br.Model)
+	if len(cands) == 0 {
 		apiError(c, http.StatusServiceUnavailable, "no_channel", "No available upstream channel")
 		g.recordError(c, nil, nil, now(), "no_channel")
 		return
 	}
-	plan, err := prepareUpstream(ch.Provider, convert.ProtoChat, body)
-	if err != nil {
-		apiError(c, http.StatusInternalServerError, "conversion_error", "Failed to convert request: "+err.Error())
-		return
-	}
 	sink := &usageSink{}
 	c.Set("usage_raw", &sinkHolder{sink: sink})
-	g.doProxy(c, ch, plan, br.Stream, sink)
+	g.doProxy(c, cands, convert.ProtoChat, body, br.Stream, sink)
 }
 
 // responses POST /v1/responses（OpenAI Responses API）
@@ -57,20 +52,15 @@ func (g *Gateway) responses(c *gin.Context) {
 	c.Set("protocol", convert.ProtoResponses)
 	c.Set("model_name", br.Model)
 
-	ch, err := g.selectChannel(c, br.Model)
-	if err != nil {
+	cands := g.candidateChannels(br.Model)
+	if len(cands) == 0 {
 		apiError(c, http.StatusServiceUnavailable, "no_channel", "No available upstream channel")
 		g.recordError(c, nil, nil, now(), "no_channel")
 		return
 	}
-	plan, err := prepareUpstream(ch.Provider, convert.ProtoResponses, body)
-	if err != nil {
-		apiError(c, http.StatusInternalServerError, "conversion_error", "Failed to convert request: "+err.Error())
-		return
-	}
 	sink := &usageSink{}
 	c.Set("usage_raw", &sinkHolder{sink: sink})
-	g.doProxy(c, ch, plan, br.Stream, sink)
+	g.doProxy(c, cands, convert.ProtoResponses, body, br.Stream, sink)
 }
 
 // messages POST /v1/messages（Anthropic Messages API）
@@ -90,20 +80,15 @@ func (g *Gateway) messages(c *gin.Context) {
 	c.Set("protocol", convert.ProtoMessages)
 	c.Set("model_name", br.Model)
 
-	ch, err := g.selectChannel(c, br.Model)
-	if err != nil {
+	cands := g.candidateChannels(br.Model)
+	if len(cands) == 0 {
 		apiError(c, http.StatusServiceUnavailable, "no_channel", "No available upstream channel")
 		g.recordError(c, nil, nil, now(), "no_channel")
 		return
 	}
-	plan, err := prepareUpstream(ch.Provider, convert.ProtoMessages, body)
-	if err != nil {
-		apiError(c, http.StatusInternalServerError, "conversion_error", "Failed to convert request: "+err.Error())
-		return
-	}
 	sink := &usageSink{}
 	c.Set("usage_raw", &sinkHolder{sink: sink})
-	g.doProxy(c, ch, plan, br.Stream, sink)
+	g.doProxy(c, cands, convert.ProtoMessages, body, br.Stream, sink)
 }
 
 // usageSinkHolder 桥接：gin context 里保存 sink 引用，供 finishUsage 读取最终 usage。

server/internal/proxy/passthrough.go

@@ -48,12 +48,45 @@ func upstreamURL(ch *store.Channel, path string) string {
 	return strings.TrimRight(ch.BaseURL, "/") + path
 }
 
-// doProxy 通用代理：替换 Authorization 为渠道密钥，转发请求；按 plan 决定路径与转换。
-func (g *Gateway) doProxy(c *gin.Context, ch *store.Channel, plan *upstreamPlan, stream bool, sink *usageSink) {
+// doProxy 通用代理（M5）：遍历候选渠道，按需转换；可安全重试的失败自动故障转移。
+func (g *Gateway) doProxy(c *gin.Context, cands []*store.Channel, clientProto string, body []byte, stream bool, sink *usageSink) {
+	var lastStatus = http.StatusBadGateway
+	var lastBody = []byte("all upstream channels failed")
+	for _, ch := range cands {
+		plan, err := prepareUpstream(ch.Provider, clientProto, body)
+		if err != nil {
+			lastStatus, lastBody = http.StatusInternalServerError, []byte("conversion error: "+err.Error())
+			continue
+		}
+		release, ok := g.ch.TryAcquire(ch)
+		if !ok {
+			continue // 渠道满载，溢出到下一个
+		}
+		written, retry, st, b := g.proxyOne(c, ch, plan, stream, sink)
+		release()
+		if written {
+			return
+		}
+		if !retry {
+			// 非重试性失败（如 400）：透传上游错误体
+			c.Header("Content-Type", "application/json")
+			c.Data(st, "application/json", b)
+			g.recordError(c, ch, nil, now(), "upstream_http_"+strconv.Itoa(st))
+			return
+		}
+		lastStatus, lastBody = st, b
+	}
+	// 全部候选重试性失败
+	apiError(c, lastStatus, "upstream_error", string(lastBody))
+	g.recordError(c, nil, nil, now(), "all_channels_failed")
+}
+
+// proxyOne 对单个渠道执行一次代理。
+// 返回：written=是否已写客户端响应；retry=是否可安全换渠道重试；status+respBody=失败信息。
+func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan, stream bool, sink *usageSink) (written bool, retry bool, status int, respBody []byte) {
 	upKey, err := g.ch.UpstreamKey(ch)
 	if err != nil {
-		apiError(c, http.StatusInternalServerError, "channel_error", "failed to decrypt channel key")
-		return
+		return false, true, http.StatusInternalServerError, []byte("failed to decrypt channel key")
 	}
 
 	upBody := plan.body
@@ -72,8 +105,7 @@ func (g *Gateway) doProxy(c *gin.Context, ch *store.Channel, plan *upstreamPlan,
 	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamURL(ch, plan.path), bytes.NewReader(upBody))
 	if err != nil {
-		apiError(c, http.StatusInternalServerError, "internal_error", "failed to build upstream request")
-		return
+		return false, false, http.StatusInternalServerError, []byte("failed to build upstream request")
 	}
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Authorization", "Bearer "+upKey)
@@ -84,7 +116,6 @@ func (g *Gateway) doProxy(c *gin.Context, ch *store.Channel, plan *upstreamPlan,
 	if plan.path == "/v1/messages" {
 		req.Header.Set("anthropic-version", "2023-06-01")
 	}
-	// 透传 OpenAI 生态请求头（组织/项目等）
 	for _, h := range []string{"OpenAI-Organization", "OpenAI-Project", "OpenAI-Beta"} {
 		if v := c.GetHeader(h); v != "" {
 			req.Header.Set(h, v)
@@ -94,29 +125,20 @@ func (g *Gateway) doProxy(c *gin.Context, ch *store.Channel, plan *upstreamPlan,
 	start := time.Now()
 	resp, err := g.hc.Do(req)
 	if err != nil {
-		status := http.StatusBadGateway
-		msg := "Upstream request failed: " + err.Error()
 		if ctx.Err() == context.DeadlineExceeded {
-			status = http.StatusGatewayTimeout
-			msg = "Upstream request timed out"
+			return false, true, http.StatusGatewayTimeout, []byte("upstream request timed out")
 		}
-		apiError(c, status, "upstream_error", msg)
-		g.recordError(c, ch, nil, start, "upstream_error")
-		return
+		return false, true, http.StatusBadGateway, []byte("upstream request failed: " + err.Error())
 	}
 	defer resp.Body.Close()
 
-	// 非 2xx：透传上游错误体，并记录 error 用量
+	// 非 2xx：429/5xx 可重试；其余透传错误体
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		errBody, _ := io.ReadAll(resp.Body)
-		status := resp.StatusCode
-		if status >= 500 {
-			status = http.StatusBadGateway
+		if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
+			return false, true, resp.StatusCode, errBody
 		}
-		c.Header("Content-Type", "application/json")
-		c.DataFromReader(status, int64(len(errBody)), "application/json", bytes.NewReader(errBody), nil)
-		g.recordError(c, ch, resp, start, "upstream_http_"+strconv.Itoa(resp.StatusCode))
-		return
+		return false, false, resp.StatusCode, errBody
 	}
 
 	c.Header("Content-Type", resp.Header.Get("Content-Type"))
@@ -126,6 +148,7 @@ func (g *Gateway) doProxy(c *gin.Context, ch *store.Channel, plan *upstreamPlan,
 	} else {
 		g.copyAndCapture(c, ch, resp.Body, start, plan.bodyConv, sink)
 	}
+	return true, false, http.StatusOK, nil
 }
 
 // copyAndCapture 非流式：整体转发（可转换）+ 解析 usage + 记账。