feat: 流式首包缓冲容错（hold）与失败分类冷却

流式请求在首个内容事件到达前由网关缓冲（借鉴 magpie 的 holdWriter）：上游的错误
事件、空流、断连、协议形态不符（要流给了整包）都在放行前换渠道消化，客户端无感知。
扣住期间照常发 SSE 注释心跳；缓冲超过 1MB 直接放行退化为普通转发，判定失效不吞正文。

- 新增 proxy/hold.go：streamHold（缓冲/放行/心跳/首 token 计时与放行互斥）、
  sseEventHasContent（chat/messages/responses 三种协议的内容判定）、
  sseUpstreamError（200+流式形式的错误事件识别）、classifyUpstreamError +
  quotaSignals（余额/配额特征识别）
- 非流式改为先读完整响应再写头：读失败（网络错误/超时）发生在写出之前，可换渠道
  重试；此前先写 200 头再读，超时只能给客户端留一个 200 空响应
- 协议形态校验：要流却给了整包（didNotStream）按失败换渠道，反复如此的渠道会被
  健康计数剔除
- 记账：每次尝试独立 trySink，只有成功走到 EOF 才并入请求级 sink——失败渠道的
  message_start 用量不计入账单
- 冷却按原因分类（channel.FailKind）：余额/配额类第一次失败即冷却 30 分钟（429 不
  一定是限流，智谱 429「余额不足」也在此列）；其余按连续进冷却次数指数退避
  30s→60s→120s，封顶 10 分钟；短于退避值的 Retry-After 不再缩短冷却
- 新增 OT_PROXY_FIRST_TOKEN_WAIT（默认 0=不限）：放行前等内容的最长时间，超时即
  取消本次尝试换下一个渠道
- 渠道列表接口返回 cooldown_reason，后台可区分「限流」与「余额耗尽」
- 测试：hold 缓冲/放行/心跳、内容与错误事件判定、失败分类与指数退避、容量热生效
- docs/magpie-plugin-mechanism.md：参考实现的插件机制调研

server/internal/proxy/passthrough.go

@@ -14,13 +14,12 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/openteam/server/internal/channel"
 	"github.com/openteam/server/internal/pkg/tokenizer"
+	"github.com/openteam/server/internal/proxy/convert"
 	"github.com/openteam/server/internal/store"
 )
 
@@ -190,6 +189,8 @@ func upstreamURL(ch *store.Channel, proto, path string) string {
 }
 
 // doProxy 通用代理（M5）：加权随机选一个候选渠道；失败自动故障转移。
+// 流式请求挂一条 streamHold：首个内容事件到达之前，上游的错误/空流/断连都还能
+// 换渠道消化，客户端无感知（借鉴 magpie 的 holdWriter）。
 func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto string, body []byte, stream bool, sink *usageSink) {
 	if len(cands) == 0 {
 		apiError(c, http.StatusBadGateway, "upstream_error", "no available channel")
@@ -206,6 +207,15 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
 		}
 	}
 
+	// 流式容错缓冲 + 心跳，整条请求一条，跨渠道重试共用
+	var sh *streamHold
+	stopHB := func() {}
+	if stream {
+		sh = newStreamHold(c)
+		stopHB = sh.heartbeat(g.streamHeartbeat)
+		defer stopHB()
+	}
+
 	var lastStatus = http.StatusBadGateway
 	var lastBody = []byte("all upstream channels failed")
 	// 最后真正试过的渠道：全部失败时把错误归到它头上，否则 usage_logs 里
@@ -243,13 +253,19 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
 		}
 		// 失败/成功计数由 proxyOne 就地报：只有它读得到上游响应头（429 的 Retry-After）
 		lastCh = ch
-		done, retry, st, b := g.proxyOne(c, ch, plan, stream, sink)
+		done, retry, st, b := g.proxyOne(c, ch, plan, stream, sink, sh, clientProto)
 		release()
 		if done {
 			return
 		}
 		if !retry {
-			// 非重试性失败（如 400）：透传上游错误体
+			// 非重试性失败（如 400）：透传上游错误体。
+			// 响应头若已被心跳先行写出，状态码不可再改，只能以 SSE 错误事件收尾。
+			if sh != nil && sh.headSent() {
+				_ = sh.writeDirect(streamErrorEvent(clientProto, st, string(b)))
+				g.recordError(c, ch, nil, now(), "upstream_http_"+strconv.Itoa(st))
+				return
+			}
 			c.Header("Content-Type", "application/json")
 			c.Data(st, "application/json", b)
 			g.recordError(c, ch, nil, now(), "upstream_http_"+strconv.Itoa(st))
@@ -258,17 +274,39 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
 		lastStatus, lastBody = st, b
 	}
 	// 全部候选重试性失败
+	if sh != nil && sh.headSent() {
+		_ = sh.writeDirect(streamErrorEvent(clientProto, lastStatus, string(lastBody)))
+		g.recordError(c, lastCh, nil, now(), "all_channels_failed")
+		return
+	}
 	apiError(c, lastStatus, "upstream_error", string(lastBody))
 	g.recordError(c, lastCh, nil, now(), "all_channels_failed")
 }
 
+// streamErrorEvent 把错误构造成客户端协议的 SSE 错误事件（响应头已写出时的唯一交付方式）。
+func streamErrorEvent(proto string, status int, msg string) []byte {
+	errObj := map[string]any{"message": msg, "type": "api_error", "code": "upstream_error", "status": status}
+	switch proto {
+	case convert.ProtoMessages:
+		body, _ := json.Marshal(map[string]any{"type": "error", "error": errObj})
+		return append([]byte("event: error\ndata: "), append(body, '\n', '\n')...)
+	case convert.ProtoResponses:
+		body, _ := json.Marshal(map[string]any{"type": "error", "code": "upstream_error", "message": msg})
+		return append([]byte("data: "), append(body, '\n', '\n')...)
+	default:
+		body, _ := json.Marshal(map[string]any{"error": errObj})
+		return append(append([]byte("data: "), append(body, '\n', '\n')...), []byte("data: [DONE]\n\n")...)
+	}
+}
+
 // proxyOne 对单个渠道执行一次代理。
 // 返回：done=本次请求已终结（客户端响应已写出或已记账，停止故障转移）；
 // retry=是否可安全换渠道重试；status+respBody=失败信息。
-func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan, stream bool, sink *usageSink) (done bool, retry bool, status int, respBody []byte) {
+// sh 为流式容错缓冲（非流式为 nil）；sink 为请求级记账 sink。
+func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan, stream bool, sink *usageSink, sh *streamHold, clientProto string) (done bool, retry bool, status int, respBody []byte) {
 	upKey, err := g.ch.UpstreamKey(ch)
 	if err != nil {
-		g.ch.ReportFailure(ch, 0)
+		g.ch.ReportFailure(ch, channel.FailNet, 0)
 		return false, true, http.StatusInternalServerError, []byte("failed to decrypt channel key")
 	}
 
@@ -294,8 +332,24 @@ func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan
 		ctx, cancel = context.WithTimeout(ctx, limit)
 		defer cancel()
 	}
+	// 首 token 计时（可选）：流式在放行前迟迟不出内容就取消本次尝试、换下一个渠道。
+	// 计时器与放行互斥（cancelIfHeld），已放行的健康流不会被掐死。
+	tryCtx := ctx
+	var tryCancel context.CancelFunc
+	var ftTimer *time.Timer
+	if stream && g.firstTokenWait > 0 {
+		tryCtx, tryCancel = context.WithCancel(ctx)
+		defer tryCancel()
+		ftTimer = time.AfterFunc(g.firstTokenWait, func() { sh.cancelIfHeld(tryCancel) })
+		defer ftTimer.Stop()
+	}
+	onRelease := func() {
+		if ftTimer != nil {
+			ftTimer.Stop()
+		}
+	}
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstreamURL(ch, plan.proto, plan.path), bytes.NewReader(upBody))
+	req, err := http.NewRequestWithContext(tryCtx, http.MethodPost, upstreamURL(ch, plan.proto, plan.path), bytes.NewReader(upBody))
 	if err != nil {
 		return false, false, http.StatusInternalServerError, []byte("failed to build upstream request")
 	}
@@ -328,46 +382,64 @@ func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan
 			g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
 			return true, false, 0, nil
 		}
-		g.ch.ReportFailure(ch, 0) // 网络错误/超时
+		g.ch.ReportFailure(ch, channel.FailNet, 0) // 网络错误/超时
 		return false, true, http.StatusBadGateway, []byte("upstream request failed: " + err.Error())
 	}
 	defer resp.Body.Close()
 
-	// 非 2xx：429/5xx 可重试；其余透传错误体
+	// 非 2xx：按原因分类记账——429 不一定是限流（可能是余额/配额），402 也值得
+	// 换渠道（渠道 A 没余额不代表渠道 B 没有）；其余透传错误体。
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		errBody, _ := io.ReadAll(resp.Body)
-		if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
-			g.ch.ReportFailure(ch, retryAfter(resp.Header))
+		errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+		kind, retryable := classifyUpstreamError(resp.StatusCode, errBody)
+		if retryable {
+			g.ch.ReportFailure(ch, kind, retryAfter(resp.Header))
 			return false, true, resp.StatusCode, errBody
 		}
 		return false, false, resp.StatusCode, errBody
 	}
 	g.ch.ReportSuccess(ch)
 
-	c.Header("Content-Type", resp.Header.Get("Content-Type"))
-	c.Status(http.StatusOK)
-	if stream {
-		g.streamCopy(c, ch, resp.Body, start, plan.lineConv, sink)
-	} else {
-		g.copyAndCapture(c, ch, resp.Body, start, plan.bodyConv, sink)
+	// 协议形态校验：要流给了整包（或反过来）是劣质中转的常见毛病，按失败换渠道；
+	// 反复如此的渠道会被健康计数剔除（didNotStream 不会因重试而变好）。
+	ct := resp.Header.Get("Content-Type")
+	isSSE := strings.HasPrefix(ct, "text/event-stream")
+	if stream != isSSE {
+		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<20)) // 排干以复用连接
+		g.ch.ReportFailure(ch, channel.FailServer, 0)
+		asked, got := "整包", "流式响应"
+		if stream {
+			asked, got = "流式", "整包响应"
+		}
+		return false, true, http.StatusBadGateway,
+			[]byte(ch.Name + " answered a " + asked + " request with " + got + " (didNotStream)")
 	}
-	return true, false, http.StatusOK, nil
+
+	if stream {
+		trySink := &usageSink{}
+		return g.streamHoldCopy(c, sh, ch, resp.Body, start, plan.lineConv, clientProto, trySink, sink, onRelease, tryCtx)
+	}
+	return g.copyAndCapture(c, ch, resp, start, plan.bodyConv, sink)
 }
 
-// copyAndCapture 非流式：整体转发（可转换）+ 解析 usage + 记账。
-func (g *Gateway) copyAndCapture(c *gin.Context, ch *store.Channel, r io.Reader, start time.Time, bodyConv func([]byte) ([]byte, error), sink *usageSink) {
-	data, err := io.ReadAll(r)
+// copyAndCapture 非流式：先读完整上游响应再写客户端——响应头与正文都在"确定成功"后
+// 才写出，读失败（网络错误/超时）发生在写出之前，可换渠道重试。
+// 此前实现先写 200 头再读，超时只能给客户端留一个 200 空响应。
+func (g *Gateway) copyAndCapture(c *gin.Context, ch *store.Channel, resp *http.Response, start time.Time, bodyConv func([]byte) ([]byte, error), sink *usageSink) (done bool, retry bool, status int, respBody []byte) {
+	data, err := io.ReadAll(resp.Body)
 	if err != nil {
-		// 走到这里响应头已按 200 写出，无法再回错误体，只按中断原因记账
 		switch {
 		case c.Request.Context().Err() != nil:
+			// 客户端断开：就地记账终结
 			g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
+			return true, false, 0, nil
 		case errors.Is(err, context.DeadlineExceeded):
-			g.finishUsage(c, ch, start, store.UsageStatusError, "upstream_timeout")
+			g.ch.ReportFailure(ch, channel.FailNet, 0)
+			return false, true, http.StatusBadGateway, []byte(ch.Name + " upstream timeout")
 		default:
-			g.finishUsage(c, ch, start, store.UsageStatusError, "read_error")
+			g.ch.ReportFailure(ch, channel.FailNet, 0)
+			return false, true, http.StatusBadGateway, []byte(ch.Name + " read error: " + err.Error())
 		}
-		return
 	}
 	if usageRaw := extractUsage(data); usageRaw != nil && sink != nil {
 		sink.push(usageRaw)
@@ -378,128 +450,105 @@ func (g *Gateway) copyAndCapture(c *gin.Context, ch *store.Channel, r io.Reader,
 			out = converted
 		}
 	}
+	c.Header("Content-Type", resp.Header.Get("Content-Type"))
+	c.Status(http.StatusOK)
 	_, _ = c.Writer.Write(out)
 	if _, ok := c.Get("raw_request"); ok {
 		c.Set("raw_response", string(data)) // 上游原始响应（未转换）
 	}
 	g.finishUsage(c, ch, start, store.UsageStatusSuccess, "")
+	return true, false, 0, nil
 }
 
-// streamCopy 流式：边读上游 SSE 边写客户端，零缓冲转发；按 lineConv 转换；扫描 usage 记账。
-func (g *Gateway) streamCopy(c *gin.Context, ch *store.Channel, r io.Reader, start time.Time, lineConv func([]byte) []byte, sink *usageSink) {
-	w := c.Writer
-	flusher, ok := w.(http.Flusher)
-	if !ok {
-		flusher = nopFlusher{}
-	}
-
+// streamHoldCopy 流式转发：边读上游 SSE 边经 hold 写客户端（放行前缓冲）。
+// 放行前的一切失败（上游错误事件、空流、断连、首 token 超时）都可换渠道重试，
+// 健康计数就地上报；放行后的失败与旧语义一致，就地记账终结。
+// trySink 收集本次尝试扫到的 usage，只有成功走到 EOF 才并入请求级 reqSink——
+// 失败渠道的 message_start 用量不能计入最终账单。
+func (g *Gateway) streamHoldCopy(c *gin.Context, sh *streamHold, ch *store.Channel, r io.Reader, start time.Time,
+	lineConv func([]byte) []byte, clientProto string, trySink, reqSink *usageSink, onRelease func(), tryCtx context.Context,
+) (done bool, retry bool, status int, respBody []byte) {
 	// 原始响应捕获：仅管理员且开关开启（raw_request 已 set）时累积上游原始行
 	_, capture := c.Get("raw_request")
 	var rawResp strings.Builder
-
-	// commitRaw 在记账前把已累积的原始响应写入 context
 	commitRaw := func() {
 		if capture {
 			c.Set("raw_response", rawResp.String())
 		}
 	}
 
-	lw := &lockedWriter{w: w, f: flusher}
-	lw.lastN.Store(time.Now().UnixNano())
-	stopHB := lw.heartbeat(g.streamHeartbeat)
-	defer stopHB()
-
 	scanner := newSSEScanner(r)
+	releaseSeen := false
+	fail := func(msg string, kind channel.FailKind) (bool, bool, int, []byte) {
+		g.ch.ReportFailure(ch, kind, 0)
+		return false, true, http.StatusBadGateway, []byte(ch.Name + ": " + msg)
+	}
 	for {
 		line, err := scanner.Next()
 		if line != nil {
 			if capture {
 				rawResp.Write(line)
 			}
+			// 上游错误事件（200+流式形式的错误）：放行前换渠道消化
+			if !releaseSeen {
+				if msg, isErr := sseUpstreamError(line); isErr {
+					return fail("stream error: "+msg, channel.FailServer)
+				}
+			}
 			out := line
 			if lineConv != nil {
 				out = lineConv(line)
 			}
 			if out != nil {
-				if _, werr := lw.write(out); werr != nil {
+				if werr := sh.writeEvent(out); werr != nil {
 					// 客户端意外断开：按已生成部分收费（canceled）
 					commitRaw()
 					g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
-					return
+					return true, false, 0, nil
 				}
-				if sink != nil {
-					sink.outputText += sseContentText(out)
+				if trySink != nil {
+					trySink.outputText += sseContentText(out)
 				}
 			}
-			if usageRaw := scanUsage(line); usageRaw != nil && sink != nil {
-				sink.push(usageRaw)
+			if !releaseSeen && sseEventHasContent(out, clientProto) {
+				releaseSeen = true
+				sh.release()
+				if onRelease != nil {
+					onRelease()
+				}
+			}
+			if usageRaw := scanUsage(line); usageRaw != nil && trySink != nil {
+				trySink.push(usageRaw)
 			}
 		}
 		if err != nil {
-			commitRaw()
-			if err == io.EOF {
-				g.finishUsage(c, ch, start, store.UsageStatusSuccess, "")
-			} else if c.Request.Context().Err() != nil {
-				// 客户端意外断开：按已生成部分收费（canceled）
-				g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
-			} else {
-				g.recordError(c, ch, nil, start, "stream_read_error")
-			}
-			return
-		}
-	}
-}
-
-type nopFlusher struct{}
-
-func (nopFlusher) Flush() {}
-
-// lockedWriter 给流式写出加锁：心跳在独立 goroutine 里写，而 gin 的 ResponseWriter
-// 不是并发安全的，两路同时写会坏帧。
-type lockedWriter struct {
-	mu    sync.Mutex
-	w     io.Writer
-	f     http.Flusher
-	lastN atomic.Int64 // 最近一次写出时间（unix nano），心跳据此判断空闲
-}
-
-func (lw *lockedWriter) write(b []byte) (int, error) {
-	lw.mu.Lock()
-	defer lw.mu.Unlock()
-	n, err := lw.w.Write(b)
-	lw.lastN.Store(time.Now().UnixNano())
-	if err == nil {
-		lw.f.Flush()
-	}
-	return n, err
-}
-
-// heartbeat 距上次写出超过 interval 时补一行 SSE 注释（`: ping`），返回停止函数。
-// 挂在 nginx/Cloudflare 后面时长思考（首包后上游安静几十秒）会被空闲超时掐掉，
-// 客户端只看到流断在半路。注释行按 SSE 规范被解析器忽略，不影响计费与转换。
-func (lw *lockedWriter) heartbeat(interval time.Duration) func() {
-	if interval <= 0 {
-		return func() {}
-	}
-	done := make(chan struct{})
-	var once sync.Once
-	go func() {
-		tk := time.NewTicker(interval)
-		defer tk.Stop()
-		for {
-			select {
-			case <-done:
-				return
-			case <-tk.C:
-				if time.Since(time.Unix(0, lw.lastN.Load())) < interval {
-					continue
+			if releaseSeen {
+				// 已放行：客户端正在看内容，失败就地记账终结，与旧语义一致
+				commitRaw()
+				if err == io.EOF {
+					reqSink.merge(trySink)
+					g.finishUsage(c, ch, start, store.UsageStatusSuccess, "")
+				} else if c.Request.Context().Err() != nil {
+					g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
+				} else {
+					g.recordError(c, ch, nil, start, "stream_read_error")
 				}
-				// 心跳写失败说明客户端已断开，交给主循环下一次写出时统一记账
-				_, _ = lw.write([]byte(": ping\n\n"))
+				return true, false, 0, nil
 			}
+			// 放行前失败：换渠道重试
+			if c.Request.Context().Err() != nil {
+				g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
+				return true, false, 0, nil
+			}
+			if tryCtx != nil && tryCtx.Err() != nil {
+				return fail(fmt.Sprintf("first token timeout (no content in %s)", g.firstTokenWait), channel.FailNet)
+			}
+			if err == io.EOF {
+				return fail("ended the stream before any content", channel.FailServer)
+			}
+			return fail("stream failed before any content: "+err.Error(), channel.FailNet)
 		}
-	}()
-	return func() { once.Do(func() { close(done) }) }
+	}
 }
 
 // ---------------------------------------------------------------------------
@@ -674,6 +723,33 @@ func (u *usageSink) push(raw json.RawMessage) {
 // Shape 返回合并后的用量。
 func (u *usageSink) Shape() usageShape { return u.us }
 
+// merge 并入一次成功尝试的用量：失败渠道的用量整体丢弃（换渠道重试时），
+// 成功渠道的 usage 事件（含放行前缓冲期扫到的 message_start）在这里落账。
+func (u *usageSink) merge(o *usageSink) {
+	if o == nil {
+		return
+	}
+	if o.us.PromptTokens > 0 {
+		u.us.PromptTokens = o.us.PromptTokens
+	}
+	if o.us.CompletionTokens > 0 {
+		u.us.CompletionTokens = o.us.CompletionTokens
+	}
+	if o.us.InputTokens > 0 {
+		u.us.InputTokens = o.us.InputTokens
+	}
+	if o.us.OutputTokens > 0 {
+		u.us.OutputTokens = o.us.OutputTokens
+	}
+	if o.us.CacheReadInputTokens > 0 {
+		u.us.CacheReadInputTokens = o.us.CacheReadInputTokens
+	}
+	if o.us.CacheCreationInputTokens > 0 {
+		u.us.CacheCreationInputTokens = o.us.CacheCreationInputTokens
+	}
+	u.outputText += o.outputText
+}
+
 // finishUsage 落账：计算成本并异步写入。
 func (g *Gateway) finishUsage(c *gin.Context, ch *store.Channel, start time.Time, status, errCode string) {
 	uid, _ := c.Get(CtxUserID)