feat: 余额预扣 reserve/capture、冷却兜底重试、Prometheus /metrics
- 预扣:每请求按 body 估算成本上界计入冻结额(1e-8 USD 原子计数),402 判定 改为「余额 - 在途冻结 ≤ 0」,并发的长流式请求不再互相透支;OT_PROXY_RESERVE_MAX_TOKENS 控制估算基数(默认 4096,0=关闭),handler defer 释放,失败/中断/panic 均覆盖 - 调度:全部候选都在运行期冷却时,剩余冷却 ≤ OT_PROXY_COOLDOWN_GRACE 的最早恢复渠道 兜底重试一次而不是 503(429 Retry-After 语义);DB 层不可用的渠道不参与兜底 - 指标:openteam_proxy_requests_total / request_duration / channel_failures / usage_queue_depth / frozen_balance,/metrics 默认关闭(OT_METRICS_ENABLED 开启)
This commit is contained in:
1 parent
b2e500ff51
commit
0e10f0bbde
14 files changed
+493
-36
No files matched your search
@@ -509,6 +509,91 @@ func TestFailoverSwitchCaps(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 预扣(reserve/capture):在途请求的冻结额让「余额 - 冻结 ≤ 0」触发 402,
|
||||
// 释放后恢复放行。并发的长流式请求不再互相透支。
|
||||
func TestReserveFreezesBalance(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.WriteString(w, `{"id":"x","choices":[{"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
h := newHarness(t, srv.URL, 0, true, nil)
|
||||
h.gw.reserveMaxTokens = 4096
|
||||
// 余额压到 0.001:4096 out × 0.6/1e6 = 0.00246 的冻结上界必然覆盖它
|
||||
if err := h.db.Model(&store.User{}).Where("username = ?", "u1").
|
||||
Update("balance", 0.001).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var u store.User
|
||||
h.db.Where("username = ?", "u1").First(&u)
|
||||
var model store.Model
|
||||
h.db.Where("name = ?", "m1").First(&model)
|
||||
|
||||
// 模拟一条在途请求的冻结(不释放)
|
||||
fr, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
fr.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
|
||||
h.gw.freezeFor(fr, u.ID, model, &bodyReq{MaxTokens: nil}, make([]byte, 64))
|
||||
if w := h.run(t, nil, `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`); w.Code != http.StatusPaymentRequired {
|
||||
t.Fatalf("冻结额覆盖余额时应 402,实际 %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// capture(释放)后恢复放行
|
||||
h.gw.unfreeze(fr, u.ID)
|
||||
if a := h.gw.frozenOf(u.ID); a != nil && a.Load() != 0 {
|
||||
t.Fatalf("释放后冻结额应为 0,实际 %d", a.Load())
|
||||
}
|
||||
w := h.run(t, nil, `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("释放后应正常代理,实际 %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// 幂等:重复 unfreeze 不产生负冻结
|
||||
h.gw.unfreeze(fr, u.ID)
|
||||
if a := h.gw.frozenOf(u.ID); a != nil && a.Load() < 0 {
|
||||
t.Fatalf("重复释放不应产生负冻结,实际 %d", a.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// 冷却兜底宽限:唯一渠道进运行期冷却后,请求 503;配置 grace 且上游已恢复时,
|
||||
// 带上冷却中的渠道兜底重试一次即成功。
|
||||
func TestCooldownGraceFallback(t *testing.T) {
|
||||
var healthy atomic.Bool
|
||||
healthy.Store(false)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !healthy.Load() {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"boom"}}`)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"id":"x","choices":[{"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
h := newHarness(t, srv.URL, 0, true, nil)
|
||||
body := `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`
|
||||
|
||||
// 连续 3 次 5xx:触发运行期冷却(failThreshold=3)
|
||||
for i := 0; i < 3; i++ {
|
||||
h.run(t, nil, body)
|
||||
}
|
||||
// 默认 grace=0(关闭):冷却中 → 无候选 → 503
|
||||
if w := h.run(t, nil, body); w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("冷却中且未配置宽限时应 503,实际 %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// 上游恢复 + 配置宽限:兜底重试冷却中的渠道 → 200
|
||||
healthy.Store(true)
|
||||
h.gw.cooldownGrace = time.Minute
|
||||
if w := h.run(t, nil, body); w.Code != http.StatusOK {
|
||||
t.Fatalf("宽限期内应兜底重试恢复的渠道,实际 %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
// 冷却截止时间未被成功清除(由 InCooldown 到期惰性恢复,语义不变),
|
||||
// 但宽限期内每次请求都能经兜底打到这条渠道
|
||||
if w := h.run(t, nil, body); w.Code != http.StatusOK {
|
||||
t.Fatalf("宽限期内后续请求也应走兜底,实际 %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 流式空闲时补 SSE 注释心跳:上游长时间不出字(长思考)不能被反代空闲超时掐掉。
|
||||
func TestStreamHeartbeatWhileIdle(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
Reference in new issue
Block a user