feat: 余额预扣 reserve/capture、冷却兜底重试、Prometheus /metrics
- 预扣:每请求按 body 估算成本上界计入冻结额(1e-8 USD 原子计数),402 判定 改为「余额 - 在途冻结 ≤ 0」,并发的长流式请求不再互相透支;OT_PROXY_RESERVE_MAX_TOKENS 控制估算基数(默认 4096,0=关闭),handler defer 释放,失败/中断/panic 均覆盖 - 调度:全部候选都在运行期冷却时,剩余冷却 ≤ OT_PROXY_COOLDOWN_GRACE 的最早恢复渠道 兜底重试一次而不是 503(429 Retry-After 语义);DB 层不可用的渠道不参与兜底 - 指标:openteam_proxy_requests_total / request_duration / channel_failures / usage_queue_depth / frozen_balance,/metrics 默认关闭(OT_METRICS_ENABLED 开启)
This commit is contained in:
1 parent
b2e500ff51
commit
0e10f0bbde
14 files changed
+493
-36
No files matched your search
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/openteam/server/internal/channel"
|
||||
"github.com/openteam/server/internal/pkg/metrics"
|
||||
"github.com/openteam/server/internal/pkg/tokenizer"
|
||||
"github.com/openteam/server/internal/proxy/convert"
|
||||
"github.com/openteam/server/internal/store"
|
||||
@@ -34,6 +35,9 @@ func newTraceID() string {
|
||||
type bodyReq struct {
|
||||
Model string `json:"model"`
|
||||
Stream bool `json:"stream"`
|
||||
// 预扣估算用(freezeFor):chat/messages 的 max_tokens、responses 的 max_output_tokens
|
||||
MaxTokens *int `json:"max_tokens"`
|
||||
MaxOutputTokens *int `json:"max_output_tokens"`
|
||||
}
|
||||
|
||||
// parseBody 读取并回填请求体,解析 model/stream。
|
||||
@@ -283,6 +287,8 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
|
||||
// 那不是「上游坏了」(502),而是「我方满了」(503 + Retry-After)。
|
||||
saturated := false
|
||||
startAll := time.Now()
|
||||
// 全冷却兜底:候选来自 EarliestRecovery 时放行冷却中的渠道(只此一次尝试机会)
|
||||
_, forceCooling := c.Get(CtxForceCooling)
|
||||
// 换渠道次数按失败类型计数:cap 到了就不再为同类失败换渠道(配置见 failover_caps)。
|
||||
switches := map[channel.FailKind]int{}
|
||||
var lastKind channel.FailKind
|
||||
@@ -311,7 +317,7 @@ func (g *Gateway) doProxy(c *gin.Context, cands []channel.Candidate, clientProto
|
||||
g.finishUsage(c, ch, now(), store.UsageStatusCanceled, "client_disconnect")
|
||||
return
|
||||
}
|
||||
if g.ch.InCooldown(ch.ID) {
|
||||
if !forceCooling && g.ch.InCooldown(ch.ID) {
|
||||
continue // 本次请求期间已被并发请求判入运行期冷却
|
||||
}
|
||||
plan, err := prepareUpstream(ch, clientProto, body, cand.UpstreamModel)
|
||||
@@ -1071,6 +1077,14 @@ func (g *Gateway) finishUsage(c *gin.Context, ch *store.Channel, start time.Time
|
||||
traceStr, _ := trace.(string)
|
||||
latency := int(time.Since(start).Milliseconds())
|
||||
|
||||
// 指标:终结计数与耗时。channel 为 "none" 表示没打到任何渠道(无候选/满载等)
|
||||
chName := "none"
|
||||
if ch != nil {
|
||||
chName = ch.Name
|
||||
}
|
||||
metrics.RequestsTotal.WithLabelValues(chName, mn, status).Inc()
|
||||
metrics.RequestDuration.WithLabelValues(chName).Observe(time.Since(start).Seconds())
|
||||
|
||||
if status == store.UsageStatusSuccess && c.Writer.Status() >= 400 {
|
||||
status = store.UsageStatusError
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user