feat: 余额预扣 reserve/capture、冷却兜底重试、Prometheus /metrics
- 预扣:每请求按 body 估算成本上界计入冻结额(1e-8 USD 原子计数),402 判定 改为「余额 - 在途冻结 ≤ 0」,并发的长流式请求不再互相透支;OT_PROXY_RESERVE_MAX_TOKENS 控制估算基数(默认 4096,0=关闭),handler defer 释放,失败/中断/panic 均覆盖 - 调度:全部候选都在运行期冷却时,剩余冷却 ≤ OT_PROXY_COOLDOWN_GRACE 的最早恢复渠道 兜底重试一次而不是 503(429 Retry-After 语义);DB 层不可用的渠道不参与兜底 - 指标:openteam_proxy_requests_total / request_duration / channel_failures / usage_queue_depth / frozen_balance,/metrics 默认关闭(OT_METRICS_ENABLED 开启)
This commit is contained in:
1 parent
b2e500ff51
commit
0e10f0bbde
14 files changed
+493
-36
No files matched your search
@@ -0,0 +1,62 @@
|
||||
// Package metrics Prometheus 指标:进程内 registry + 文本端点(api 层按开关挂载 /metrics)。
|
||||
//
|
||||
// label 基数约束:channel/model 来自受管目录、status/kind 为有限枚举,可安全做 label;
|
||||
// 请求级随机值(trace id、用户 id、错误详情)禁止入 label,否则基数爆炸拖垮采集端。
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var reg = prometheus.NewRegistry()
|
||||
|
||||
var (
|
||||
// RequestsTotal 代理请求终结计数。channel 为 "none" 表示未打到任何渠道
|
||||
// (无候选/满载等),model 为请求的模型名。
|
||||
RequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "openteam_proxy_requests_total",
|
||||
Help: "Proxied requests terminated, by channel/model/status.",
|
||||
}, []string{"channel", "model", "status"})
|
||||
|
||||
// RequestDuration 请求从进入 handler 到终结的耗时(按渠道)。
|
||||
RequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "openteam_proxy_request_duration_seconds",
|
||||
Help: "Proxy request duration, by channel.",
|
||||
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120, 300},
|
||||
}, []string{"channel"})
|
||||
|
||||
// ChannelFailures 上游失败计数(按渠道与失败类型),与运行期冷却同源。
|
||||
ChannelFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "openteam_channel_failures_total",
|
||||
Help: "Upstream failures, by channel/fail kind.",
|
||||
}, []string{"channel", "kind"})
|
||||
|
||||
// RecorderQueueDepth 记账队列中待落库的记录数。持续增长说明 DB 写入跟不上。
|
||||
RecorderQueueDepth = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "openteam_usage_queue_depth",
|
||||
Help: "Usage records buffered, waiting to be flushed.",
|
||||
})
|
||||
|
||||
// FrozenBalance 在途请求预扣(冻结)的金额合计(USD)。见 proxy 的 reserve/capture。
|
||||
FrozenBalance = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "openteam_frozen_balance",
|
||||
Help: "Balance reserved by in-flight requests (USD).",
|
||||
})
|
||||
)
|
||||
|
||||
func init() {
|
||||
reg.MustRegister(
|
||||
RequestsTotal, RequestDuration, ChannelFailures, RecorderQueueDepth, FrozenBalance,
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
}
|
||||
|
||||
// Handler /metrics 的 http.Handler(独立 registry,避免全局默认注册器的耦合)。
|
||||
func Handler() http.Handler {
|
||||
return promhttp.HandlerFor(reg, promhttp.HandlerOpts{})
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// /metrics 文本端点:注入计数/观测后可被抓取,指标名与 label 齐全。
|
||||
func TestHandlerExposesInstruments(t *testing.T) {
|
||||
RequestsTotal.WithLabelValues("c1", "m1", "success").Inc()
|
||||
RequestsTotal.WithLabelValues("none", "m1", "error").Inc()
|
||||
RequestDuration.WithLabelValues("c1").Observe(0.42)
|
||||
ChannelFailures.WithLabelValues("c1", "rate_limited").Inc()
|
||||
RecorderQueueDepth.Set(7)
|
||||
FrozenBalance.Set(0.00246)
|
||||
|
||||
srv := httptest.NewServer(Handler())
|
||||
defer srv.Close()
|
||||
resp, err := http.Get(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d", resp.StatusCode)
|
||||
}
|
||||
buf := make([]byte, 1<<16)
|
||||
n, _ := resp.Body.Read(buf)
|
||||
body := string(buf[:n])
|
||||
for _, want := range []string{
|
||||
"openteam_proxy_requests_total{",
|
||||
`channel="c1"`,
|
||||
`status="success"`,
|
||||
"openteam_proxy_request_duration_seconds",
|
||||
"openteam_channel_failures_total",
|
||||
"openteam_usage_queue_depth 7",
|
||||
"openteam_frozen_balance 0.00246",
|
||||
"go_goroutines", // Go 运行时指标一并暴露
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("metrics 输出缺少 %q", want)
|
||||
}
|
||||
}
|
||||
// 直方图观测应产生 bucket 样本
|
||||
if !strings.Contains(body, `openteam_proxy_request_duration_seconds_bucket`) && n < len(buf) {
|
||||
// 一次 Read 可能读不完,重读剩余部分再断言
|
||||
rest := make([]byte, 1<<16)
|
||||
n2, _ := resp.Body.Read(rest)
|
||||
body += string(rest[:n2])
|
||||
if !strings.Contains(body, "openteam_proxy_request_duration_seconds_bucket") {
|
||||
t.Errorf("metrics 输出缺少直方图 bucket 样本")
|
||||
}
|
||||
}
|
||||
_ = time.Second
|
||||
}
|
||||
Reference in new issue
Block a user