mirrors2/openteam
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 会话亲和粘住缓存渠道；透传 opaque 字段；用量记思考/1h 缓存写/实际模型

Browse Source 
会话亲和（proxy/affinity.go）：同一会话粘住上次应答的渠道，命中上游提示词
缓存而非被加权随机打散后全价重算。会话身份 = 密钥 + 模型 + 请求文本稳定前缀
摘要，不依赖各客户端会话头；TTL 默认 5m（OT_PROXY_AFFINITY_TTL，0=关闭），
成功应答刷新。亲和渠道必须在候选集内（健康/未冷却/绑定），否则回退加权随机。
proxyOne 增加 served 返回值区分「应答成功」与「已终结但失败」。

opaque 透传（convert）：chat↔responses 互转原样携带 metadata、service_tier、
parallel_tool_calls。此前逐字段重建请求体会丢它们，而中转上游会校验/使用。

用量细化（proxy+store）：usage_logs 新增 reasoning_tokens（展示口径，不重复
计费）、cache_write_1h_tokens（Anthropic 1h 缓存写按 2× 输入价，此前统一
1.25× 少算 60%）、served_model（上游实际应答模型，映射/聚合商场景对账）。
schema v6→v7。

验证：go vet ./... 干净；go test ./... 与 go test -race ./... 全绿；
新增测试覆盖亲和粘性与冷却回退、opaque 双向透传、reasoning 不重复计费、
1h/5m 缓存写分开计价、served_model 提取（非流式与流式）。
This commit is contained in:
Sakurasan committed 2026-10-30 11:41:51 +08:00
1 parent e7f8a5e5ad
commit 04dfb4285d
14 files changed
+541 -38
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/internal/proxy/passthrough.go
+165 -33
View File
@@ -197,13 +197,28 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
g.recordError(c, nil, nil, now(), "no_available_channel")
return
}
// 加权随机选择起始渠道
picked := g.ch.Pick(cands)
startIdx := 0
for i, cand := range cands {
if cand.Channel.ID == picked.ID {
startIdx = i
break
// 会话亲和：上次应答过该会话的渠道优先（候选集已过滤健康/冷却/绑定），
// 让上游提示词缓存被再次命中；命中不了（首次/冷却/下线）走加权随机。
affKey := g.affinityKeyFor(c)
startIdx := -1
if affKey != "" {
if chID, ok := g.aff.get(affKey); ok {
for i, cand := range cands {
if cand.Channel.ID == chID {
startIdx = i
break
}
}
}
}
if startIdx < 0 {
// 加权随机选择起始渠道
picked := g.ch.Pick(cands)
for i, cand := range cands {
if cand.Channel.ID == picked.ID {
startIdx = i
break
}
}
}
@@ -253,9 +268,12 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
}
// 失败/成功计数由 proxyOne 就地报：只有它读得到上游响应头（429 的 Retry-After）
lastCh = ch
done, retry, st, b := g.proxyOne(c, ch, plan, stream, sink, sh, clientProto)
done, served, retry, st, b := g.proxyOne(c, ch, plan, stream, sink, sh, clientProto)
release()
if done {
if served && affKey != "" {
g.aff.put(affKey, ch.ID, g.affinityTTL)
}
return
}
if !retry {
@@ -283,6 +301,23 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
g.recordError(c, lastCh, nil, now(), "all_channels_failed")
}
// affinityKeyFor 计算本请求的会话亲和键；亲和关闭或缺少身份要素时返回空。
func (g *Gateway) affinityKeyFor(c *gin.Context) string {
if g.affinityTTL <= 0 {
return ""
}
kid, ok := c.Get(CtxKeyID)
kID, ok2 := kid.(uint64)
if !ok || !ok2 {
return ""
}
mn, _ := c.Get("model_name")
model, _ := mn.(string)
text, _ := c.Get("est_input_text")
firstText, _ := text.(string)
return affinityKey(kID, model, firstText)
}
// streamErrorEvent 把错误构造成客户端协议的 SSE 错误事件（响应头已写出时的唯一交付方式）。
func streamErrorEvent(proto string, status int, msg string) []byte {
errObj := map[string]any{"message": msg, "type": "api_error", "code": "upstream_error", "status": status}
@@ -301,13 +336,14 @@ func streamErrorEvent(proto string, status int, msg string) []byte {
// proxyOne 对单个渠道执行一次代理。
// 返回：done=本次请求已终结（客户端响应已写出或已记账，停止故障转移）；
// retry=是否可安全换渠道重试；status+respBody=失败信息。
// served=该渠道成功应答（会话亲和据此绑定）；retry=是否可安全换渠道重试；
// status+respBody=失败信息。
// sh 为流式容错缓冲（非流式为 nil）；sink 为请求级记账 sink。
func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan, stream bool, sink *usageSink, sh *streamHold, clientProto string) (done bool, retry bool, status int, respBody []byte) {
func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan, stream bool, sink *usageSink, sh *streamHold, clientProto string) (done bool, served bool, retry bool, status int, respBody []byte) {
upKey, err := g.ch.UpstreamKey(ch)
if err != nil {
g.ch.ReportFailure(ch, channel.FailNet, 0)
return false, true, http.StatusInternalServerError, []byte("failed to decrypt channel key")
return false, false, true, http.StatusInternalServerError, []byte("failed to decrypt channel key")
}
upBody := plan.body
@@ -351,7 +387,7 @@ func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan
req, err := http.NewRequestWithContext(tryCtx, http.MethodPost, upstreamURL(ch, plan.proto, plan.path), bytes.NewReader(upBody))
if err != nil {
return false, false, http.StatusInternalServerError, []byte("failed to build upstream request")
return false, false, false, http.StatusInternalServerError, []byte("failed to build upstream request")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+upKey)
@@ -380,10 +416,10 @@ func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan
// 客户端断开导致的取消：就地记账并终结本次请求（不计入渠道失败，也不换渠道）
if c.Request.Context().Err() != nil {
g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
return true, false, 0, nil
return true, false, false, 0, nil
}
g.ch.ReportFailure(ch, channel.FailNet, 0) // 网络错误/超时
return false, true, http.StatusBadGateway, []byte("upstream request failed: " + err.Error())
return false, false, true, http.StatusBadGateway, []byte("upstream request failed: " + err.Error())
}
defer resp.Body.Close()
@@ -394,9 +430,9 @@ func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan
kind, retryable := classifyUpstreamError(resp.StatusCode, errBody)
if retryable {
g.ch.ReportFailure(ch, kind, retryAfter(resp.Header))
return false, true, resp.StatusCode, errBody
return false, false, true, resp.StatusCode, errBody
}
return false, false, resp.StatusCode, errBody
return false, false, false, resp.StatusCode, errBody
}
g.ch.ReportSuccess(ch)
@@ -411,7 +447,7 @@ func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan
if stream {
asked, got = "流式", "整包响应"
}
return false, true, http.StatusBadGateway,
return false, false, true, http.StatusBadGateway,
[]byte(ch.Name + " answered a " + asked + " request with " + got + " (didNotStream)")
}
@@ -425,25 +461,28 @@ func (g *Gateway) proxyOne(c *gin.Context, ch *store.Channel, plan *upstreamPlan
// copyAndCapture 非流式：先读完整上游响应再写客户端——响应头与正文都在"确定成功"后
// 才写出，读失败（网络错误/超时）发生在写出之前，可换渠道重试。
// 此前实现先写 200 头再读，超时只能给客户端留一个 200 空响应。
func (g *Gateway) copyAndCapture(c *gin.Context, ch *store.Channel, resp *http.Response, start time.Time, bodyConv func([]byte) ([]byte, error), sink *usageSink) (done bool, retry bool, status int, respBody []byte) {
func (g *Gateway) copyAndCapture(c *gin.Context, ch *store.Channel, resp *http.Response, start time.Time, bodyConv func([]byte) ([]byte, error), sink *usageSink) (done bool, served bool, retry bool, status int, respBody []byte) {
data, err := io.ReadAll(resp.Body)
if err != nil {
switch {
case c.Request.Context().Err() != nil:
// 客户端断开：就地记账终结
g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
return true, false, 0, nil
return true, false, false, 0, nil
case errors.Is(err, context.DeadlineExceeded):
g.ch.ReportFailure(ch, channel.FailNet, 0)
return false, true, http.StatusBadGateway, []byte(ch.Name + " upstream timeout")
return false, false, true, http.StatusBadGateway, []byte(ch.Name + " upstream timeout")
default:
g.ch.ReportFailure(ch, channel.FailNet, 0)
return false, true, http.StatusBadGateway, []byte(ch.Name + " read error: " + err.Error())
return false, false, true, http.StatusBadGateway, []byte(ch.Name + " read error: " + err.Error())
}
}
if usageRaw := extractUsage(data); usageRaw != nil && sink != nil {
sink.push(usageRaw)
}
if sink != nil && sink.servedModel == "" {
sink.servedModel = extractServedModel(data)
}
out := data
if bodyConv != nil {
if converted, cerr := bodyConv(data); cerr == nil {
@@ -457,7 +496,7 @@ func (g *Gateway) copyAndCapture(c *gin.Context, ch *store.Channel, resp *http.R
c.Set("raw_response", string(data)) // 上游原始响应（未转换）
}
g.finishUsage(c, ch, start, store.UsageStatusSuccess, "")
return true, false, 0, nil
return true, true, false, 0, nil
}
// streamHoldCopy 流式转发：边读上游 SSE 边经 hold 写客户端（放行前缓冲）。
@@ -467,7 +506,7 @@ func (g *Gateway) copyAndCapture(c *gin.Context, ch *store.Channel, resp *http.R
// 失败渠道的 message_start 用量不能计入最终账单。
func (g *Gateway) streamHoldCopy(c *gin.Context, sh *streamHold, ch *store.Channel, r io.Reader, start time.Time,
lineConv func([]byte) []byte, clientProto string, trySink, reqSink *usageSink, onRelease func(), tryCtx context.Context,
) (done bool, retry bool, status int, respBody []byte) {
) (done bool, served bool, retry bool, status int, respBody []byte) {
// 原始响应捕获：仅管理员且开关开启（raw_request 已 set）时累积上游原始行
_, capture := c.Get("raw_request")
var rawResp strings.Builder
@@ -479,9 +518,9 @@ func (g *Gateway) streamHoldCopy(c *gin.Context, sh *streamHold, ch *store.Chann
scanner := newSSEScanner(r)
releaseSeen := false
fail := func(msg string, kind channel.FailKind) (bool, bool, int, []byte) {
fail := func(msg string, kind channel.FailKind) (bool, bool, bool, int, []byte) {
g.ch.ReportFailure(ch, kind, 0)
return false, true, http.StatusBadGateway, []byte(ch.Name + ": " + msg)
return false, false, true, http.StatusBadGateway, []byte(ch.Name + ": " + msg)
}
for {
line, err := scanner.Next()
@@ -504,7 +543,7 @@ func (g *Gateway) streamHoldCopy(c *gin.Context, sh *streamHold, ch *store.Chann
// 客户端意外断开：按已生成部分收费（canceled）
commitRaw()
g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
return true, false, 0, nil
return true, false, false, 0, nil
}
if trySink != nil {
trySink.outputText += sseContentText(out)
@@ -520,6 +559,11 @@ func (g *Gateway) streamHoldCopy(c *gin.Context, sh *streamHold, ch *store.Chann
if usageRaw := scanUsage(line); usageRaw != nil && trySink != nil {
trySink.push(usageRaw)
}
if trySink != nil && trySink.servedModel == "" && bytes.Contains(line, []byte(`"model"`)) {
if payload, ok := sseDataPayload(line); ok {
trySink.servedModel = extractServedModel([]byte(payload))
}
}
}
if err != nil {
if releaseSeen {
@@ -533,12 +577,12 @@ func (g *Gateway) streamHoldCopy(c *gin.Context, sh *streamHold, ch *store.Chann
} else {
g.recordError(c, ch, nil, start, "stream_read_error")
}
return true, false, 0, nil
return true, err == io.EOF, false, 0, nil
}
// 放行前失败：换渠道重试
if c.Request.Context().Err() != nil {
g.finishUsage(c, ch, start, store.UsageStatusCanceled, "client_disconnect")
return true, false, 0, nil
return true, false, false, 0, nil
}
if tryCtx != nil && tryCtx.Err() != nil {
return fail(fmt.Sprintf("first token timeout (no content in %s)", g.firstTokenWait), channel.FailNet)
@@ -563,6 +607,22 @@ type usageShape struct {
TotalTokens int64 `json:"total_tokens"`
CacheReadInputTokens int64 `json:"cache_read_input_tokens"`
CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`
// Reasoning：思考 token（chat 的 completion_tokens_details / responses 的
// output_tokens_details）。上游已把它计入 completion/output，只做展示口径，不重复计费。
CompletionDetails *usageTokenDetails `json:"completion_tokens_details,omitempty"`
OutputDetails *usageTokenDetails `json:"output_tokens_details,omitempty"`
// Anthropic 的 cache_creation 细分：1h 写价是输入价 2×、5m 是 1.25×，必须拆开算钱
CacheCreation *usageCacheCreation `json:"cache_creation,omitempty"`
}
type usageTokenDetails struct {
ReasoningTokens int64 `json:"reasoning_tokens"`
}
type usageCacheCreation struct {
Ephemeral5m int64 `json:"ephemeral_5m_input_tokens"`
Ephemeral1h int64 `json:"ephemeral_1h_input_tokens"`
}
// extractUsage 从完整响应体提取 usage 子对象（chat / responses / messages）。
@@ -676,11 +736,45 @@ func (s *sseScanner) Next() ([]byte, error) {
// ---------------------------------------------------------------------------
// 记账
// extractServedModel 从响应体（或流式事件载荷）提取上游实际应答的模型名：
// chat/responses/messages 三协议都在顶层带 model；流式事件在 message.model /
// response.model。模型映射、聚合商场景下它与请求的模型名不同，落库便于对账。
func extractServedModel(data []byte) string {
var m map[string]json.RawMessage
if json.Unmarshal(data, &m) != nil {
return ""
}
if raw, ok := m["model"]; ok {
var s string
if json.Unmarshal(raw, &s) == nil && s != "" {
return s
}
}
for _, key := range []string{"response", "message"} {
raw, ok := m[key]
if !ok {
continue
}
var sub map[string]json.RawMessage
if json.Unmarshal(raw, &sub) != nil {
continue
}
if mraw, ok := sub["model"]; ok {
var s string
if json.Unmarshal(mraw, &s) == nil && s != "" {
return s
}
}
}
return ""
}
// usageSink 累积多次 usage：合并各事件字段（message_start 给 input，message_delta 给 output）。
// outputText 累积已转发内容文本，用于流式中断时按 tiktoken 估算输出 token。
type usageSink struct {
us usageShape
outputText string
us usageShape
outputText string
servedModel string // 上游实际应答的模型名（模型映射/聚合商场景下与请求的不同）
}
func (u *usageSink) push(raw json.RawMessage) {
@@ -718,6 +812,15 @@ func (u *usageSink) push(raw json.RawMessage) {
if t.CacheCreationInputTokens > 0 {
u.us.CacheCreationInputTokens = t.CacheCreationInputTokens
}
if t.CompletionDetails != nil {
u.us.CompletionDetails = t.CompletionDetails
}
if t.OutputDetails != nil {
u.us.OutputDetails = t.OutputDetails
}
if t.CacheCreation != nil {
u.us.CacheCreation = t.CacheCreation
}
}
// Shape 返回合并后的用量。
@@ -748,6 +851,9 @@ func (u *usageSink) merge(o *usageSink) {
u.us.CacheCreationInputTokens = o.us.CacheCreationInputTokens
}
u.outputText += o.outputText
if o.servedModel != "" && u.servedModel == "" {
u.servedModel = o.servedModel
}
}
// finishUsage 落账：计算成本并异步写入。
@@ -769,9 +875,30 @@ func (g *Gateway) finishUsage(c *gin.Context, ch *store.Channel, start time.Time
out := us.CompletionTokens + us.OutputTokens
cacheRead := us.CacheReadInputTokens
cacheCreate := us.CacheCreationInputTokens
reasoning := int64(0)
if us.CompletionDetails != nil {
reasoning += us.CompletionDetails.ReasoningTokens
}
if us.OutputDetails != nil {
reasoning += us.OutputDetails.ReasoningTokens
}
// Anthropic 缓存写细分：1h 是输入价 2×、5m 是 1.25×，倍率 1.6。
// CacheWritePriceOr() 是 5m 口径（缺省输入价 ×1.25），1h 按同比例放大。
cache1h := int64(0)
if us.CacheCreation != nil && us.CacheCreation.Ephemeral1h > 0 {
cache1h = us.CacheCreation.Ephemeral1h
if cache1h > cacheCreate {
cache1h = cacheCreate
}
}
cache5m := cacheCreate - cache1h
modelName, _ := c.Get("model_name")
mn, _ := modelName.(string)
servedModel := ""
if sink != nil {
servedModel = sink.servedModel
}
// 上游未返回真实 usage 时估算（tiktoken）：
// - 流式中断（canceled）：上游最终 usage 可能未返回
@@ -800,10 +927,12 @@ func (g *Gateway) finishUsage(c *gin.Context, ch *store.Channel, start time.Time
// 计价口径：in=非缓存输入、cacheRead=缓存读、cacheCreate=缓存写（Anthropic 语义，
// messages 流式 message_delta 的 input_tokens 即非缓存部分）。
// 缓存写价未单独配置时按输入价 1.25×（Anthropic 5m 口径）。
cwp := model.CacheWritePriceOr()
cost = float64(in)/1e6*model.InputPrice +
float64(out)/1e6*model.OutputPrice +
float64(cacheRead)/1e6*model.CacheReadPrice +
float64(cacheCreate)/1e6*model.CacheWritePriceOr()
float64(cache5m)/1e6*cwp +
float64(cache1h)/1e6*cwp*1.6
} else {
// 模型在记账时已被删除（请求进行中被下架）：无法定价，按 0 记账留痕。
// 虚构单价扣用户的钱既不可审计也无法对账。
@@ -871,6 +1000,9 @@ func (g *Gateway) finishUsage(c *gin.Context, ch *store.Channel, start time.Time
OutputPrice: model.OutputPrice,
CacheReadPrice: model.CacheReadPrice,
Cost: cost,
ReasoningTokens: reasoning,
CacheWrite1hTokens: cache1h,
ServedModel: servedModel,
LatencyMS: latency,
Status: status,
ErrorCode: errCodePtr,
@@ -880,8 +1012,8 @@ func (g *Gateway) finishUsage(c *gin.Context, ch *store.Channel, start time.Time
})
// 请求级一行日志：usage_logs 落库是异步的且可能丢，排障时日志是唯一即时依据
log.Printf("proxy: trace=%s user=%d key=%d model=%q channel=%d proto=%s in=%d out=%d cache_r=%d cache_w=%d cost=%.6f status=%s http=%d latency=%dms err=%s",
traceStr, uidVal, kidVal, mn, chID, p, in, out, cacheRead, cacheCreate, cost, status,
log.Printf("proxy: trace=%s user=%d key=%d model=%q served=%q channel=%d proto=%s in=%d out=%d r=%d cache_r=%d cache_w=%d(1h=%d) cost=%.6f status=%s http=%d latency=%dms err=%s",
traceStr, uidVal, kidVal, mn, servedModel, chID, p, in, out, reasoning, cacheRead, cacheCreate, cache1h, cost, status,
c.Writer.Status(), latency, errCode)
}