feat: 余额预扣 reserve/capture、冷却兜底重试、Prometheus /metrics

- 预扣:每请求按 body 估算成本上界计入冻结额(1e-8 USD 原子计数),402 判定
  改为「余额 - 在途冻结 ≤ 0」,并发的长流式请求不再互相透支;OT_PROXY_RESERVE_MAX_TOKENS
  控制估算基数(默认 4096,0=关闭),handler defer 释放,失败/中断/panic 均覆盖
- 调度:全部候选都在运行期冷却时,剩余冷却 ≤ OT_PROXY_COOLDOWN_GRACE 的最早恢复渠道
  兜底重试一次而不是 503(429 Retry-After 语义);DB 层不可用的渠道不参与兜底
- 指标:openteam_proxy_requests_total / request_duration / channel_failures /
  usage_queue_depth / frozen_balance,/metrics 默认关闭(OT_METRICS_ENABLED 开启)
This commit is contained in:
Sakurasan committed 2026-11-09 01:04:40 +08:00
1 parent b2e500ff51
commit 0e10f0bbde
14 files changed
+493 -36

No files matched your search

+62
View File
@@ -0,0 +1,62 @@
// Package metrics Prometheus 指标:进程内 registry + 文本端点(api 层按开关挂载 /metrics)。
//
// label 基数约束:channel/model 来自受管目录、status/kind 为有限枚举,可安全做 label;
// 请求级随机值(trace id、用户 id、错误详情)禁止入 label,否则基数爆炸拖垮采集端。
package metrics
import (
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var reg = prometheus.NewRegistry()
var (
// RequestsTotal 代理请求终结计数。channel 为 "none" 表示未打到任何渠道
// (无候选/满载等),model 为请求的模型名。
RequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "openteam_proxy_requests_total",
Help: "Proxied requests terminated, by channel/model/status.",
}, []string{"channel", "model", "status"})
// RequestDuration 请求从进入 handler 到终结的耗时(按渠道)。
RequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "openteam_proxy_request_duration_seconds",
Help: "Proxy request duration, by channel.",
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120, 300},
}, []string{"channel"})
// ChannelFailures 上游失败计数(按渠道与失败类型),与运行期冷却同源。
ChannelFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "openteam_channel_failures_total",
Help: "Upstream failures, by channel/fail kind.",
}, []string{"channel", "kind"})
// RecorderQueueDepth 记账队列中待落库的记录数。持续增长说明 DB 写入跟不上。
RecorderQueueDepth = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "openteam_usage_queue_depth",
Help: "Usage records buffered, waiting to be flushed.",
})
// FrozenBalance 在途请求预扣(冻结)的金额合计(USD)。见 proxy 的 reserve/capture。
FrozenBalance = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "openteam_frozen_balance",
Help: "Balance reserved by in-flight requests (USD).",
})
)
func init() {
reg.MustRegister(
RequestsTotal, RequestDuration, ChannelFailures, RecorderQueueDepth, FrozenBalance,
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
)
}
// Handler /metrics 的 http.Handler(独立 registry,避免全局默认注册器的耦合)。
func Handler() http.Handler {
return promhttp.HandlerFor(reg, promhttp.HandlerOpts{})
}
@@ -0,0 +1,58 @@
package metrics
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// /metrics 文本端点:注入计数/观测后可被抓取,指标名与 label 齐全。
func TestHandlerExposesInstruments(t *testing.T) {
RequestsTotal.WithLabelValues("c1", "m1", "success").Inc()
RequestsTotal.WithLabelValues("none", "m1", "error").Inc()
RequestDuration.WithLabelValues("c1").Observe(0.42)
ChannelFailures.WithLabelValues("c1", "rate_limited").Inc()
RecorderQueueDepth.Set(7)
FrozenBalance.Set(0.00246)
srv := httptest.NewServer(Handler())
defer srv.Close()
resp, err := http.Get(srv.URL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d", resp.StatusCode)
}
buf := make([]byte, 1<<16)
n, _ := resp.Body.Read(buf)
body := string(buf[:n])
for _, want := range []string{
"openteam_proxy_requests_total{",
`channel="c1"`,
`status="success"`,
"openteam_proxy_request_duration_seconds",
"openteam_channel_failures_total",
"openteam_usage_queue_depth 7",
"openteam_frozen_balance 0.00246",
"go_goroutines", // Go 运行时指标一并暴露
} {
if !strings.Contains(body, want) {
t.Errorf("metrics 输出缺少 %q", want)
}
}
// 直方图观测应产生 bucket 样本
if !strings.Contains(body, `openteam_proxy_request_duration_seconds_bucket`) && n < len(buf) {
// 一次 Read 可能读不完,重读剩余部分再断言
rest := make([]byte, 1<<16)
n2, _ := resp.Body.Read(rest)
body += string(rest[:n2])
if !strings.Contains(body, "openteam_proxy_request_duration_seconds_bucket") {
t.Errorf("metrics 输出缺少直方图 bucket 样本")
}
}
_ = time.Second
}