feat: 金额 decimal 精确计费、failover 按失败类型封顶、渠道小时桶预聚合、分层守护测试
- 计费：19 个金额字段 float64→decimal.Decimal，DB 列保持 numeric(20,8) 零迁移； API 金额仍为 JSON number（MarshalJSONWithoutQuotes），前端零改动 - failover：保留时间预算，新增 OT_PROXY_FAILOVER_CAPS_NET/SERVER/RATE/QUOTA 按失败类型封顶换渠道次数（429/配额类瞬时失败时间预算形同虚设） - 预聚合：新增 usage_channel_hourlies（schema v10），记账事务内同步 upsert， 长窗口渠道统计走桶不再扫明细，spent_last_30d 改扫 usage_daily，v10 一次性回填幂等 - 守护：internal/arch_test.go 用 go/parser 强制分层依赖（fail-closed）
This commit is contained in:
Sakurasan committed
1 parent
5d6e5bdb04
commit
b2e500ff51
25 files changed
+841
-253
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -14,11 +15,13 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/openteam/server/internal/channel"
|
||||
"github.com/openteam/server/internal/pkg/apikey"
|
||||
"github.com/openteam/server/internal/pkg/crypto"
|
||||
"github.com/openteam/server/internal/pkg/ratelimit"
|
||||
"github.com/openteam/server/internal/store"
|
||||
"github.com/openteam/server/internal/usage"
|
||||
"github.com/shopspring/decimal"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -46,7 +49,7 @@ func newHarness(t *testing.T, baseURL string, timeoutMS int, bind bool, keyModel
|
||||
enc := crypto.NewEncryptor("test-master-key-1234567890")
|
||||
encKey, _ := enc.Encrypt("upstream-key")
|
||||
u := store.User{Username: "u1", Email: "u1@x.com", PasswordHash: "x", Role: store.RoleUser,
|
||||
Balance: 100, Status: store.UserStatusActive}
|
||||
Balance: decimal.NewFromInt(100), Status: store.UserStatusActive}
|
||||
if err := db.Create(&u).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -56,7 +59,7 @@ func newHarness(t *testing.T, baseURL string, timeoutMS int, bind bool, keyModel
|
||||
if err := db.Create(&k).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := store.Model{Name: "m1", InputPrice: 0.15, OutputPrice: 0.6, Enabled: true}
|
||||
m := store.Model{Name: "m1", InputPrice: decimal.NewFromFloat(0.15), OutputPrice: decimal.NewFromFloat(0.6), Enabled: true}
|
||||
if err := db.Create(&m).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -426,6 +429,86 @@ func TestFailoverBudgetStopsRetrying(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 429/配额类失败瞬时返回，时间预算对它们形同虚设；按失败类型封顶换渠道次数。
|
||||
// rate 上限 2 → 首渠道 + 至多 2 次换渠道 = 3 次尝试即止；上限 0 = 不限（4 条全试）。
|
||||
func TestFailoverSwitchCaps(t *testing.T) {
|
||||
var hits atomic.Int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"rate limited, try again"}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
newMultiChannel := func(t *testing.T, n int) *harness {
|
||||
t.Helper()
|
||||
h := newHarness(t, srv.URL, 0, true, nil)
|
||||
var m store.Model
|
||||
h.db.Where("name = ?", "m1").First(&m)
|
||||
encKey, _ := crypto.NewEncryptor("test-master-key-1234567890").Encrypt("upstream-key")
|
||||
for i := 2; i <= n; i++ {
|
||||
ch := store.Channel{Name: fmt.Sprintf("c%d", i), Provider: store.ChannelProviderOpenAI,
|
||||
Formats: []string{store.FormatChat}, BaseURL: srv.URL, APIKeyEnc: encKey,
|
||||
Weight: 1, TimeoutMS: 0, MaxConcurrency: 4,
|
||||
HealthStatus: store.ChannelHealthHealthy, Enabled: true}
|
||||
if err := h.db.Create(&ch).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.db.Create(&store.ChannelModelBinding{ChannelID: ch.ID, ModelID: m.ID, UpstreamModel: "m1"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
body := `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`
|
||||
|
||||
// rate 上限 2：打满即停，第 4 条渠道不应被碰
|
||||
h := newMultiChannel(t, 4)
|
||||
h.gw.failoverCaps = map[channel.FailKind]int{channel.FailRate: 2}
|
||||
w := h.run(t, nil, body)
|
||||
if hits.Load() != 3 {
|
||||
t.Fatalf("rate 上限 2 应恰好尝试 3 条渠道（1+2），实际 %d", hits.Load())
|
||||
}
|
||||
if w.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("应透传最后的 429，实际 %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "failover switch cap reached (rate_limited)") {
|
||||
t.Fatalf("错误体应说明按类型封顶：%s", w.Body.String())
|
||||
}
|
||||
|
||||
// 上限 0 = 不限：4 条渠道全部试完
|
||||
hits.Store(0)
|
||||
h.gw.failoverCaps = map[channel.FailKind]int{channel.FailRate: 0}
|
||||
if w := h.run(t, nil, body); w.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("应 429，实际 %d", w.Code)
|
||||
}
|
||||
if hits.Load() != 4 {
|
||||
t.Fatalf("不限次时应尝试全部 4 条渠道，实际 %d", hits.Load())
|
||||
}
|
||||
|
||||
// quota 上限 1：402 打满即停（402 归为 FailQuota，第一次失败即长冷却，但换渠道仍受上限约束）
|
||||
hits.Store(0)
|
||||
h2 := newMultiChannel(t, 4)
|
||||
h2.gw.failoverCaps = map[channel.FailKind]int{channel.FailQuota: 1}
|
||||
quotaSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.WriteHeader(http.StatusPaymentRequired)
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"insufficient balance"}}`)
|
||||
}))
|
||||
defer quotaSrv.Close()
|
||||
if err := h2.db.Model(&store.Channel{}).Where("1=1").Update("base_url", quotaSrv.URL).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w2 := h2.run(t, nil, body)
|
||||
if hits.Load() != 2 {
|
||||
t.Fatalf("quota 上限 1 应恰好尝试 2 条渠道，实际 %d", hits.Load())
|
||||
}
|
||||
if w2.Code != http.StatusPaymentRequired {
|
||||
t.Fatalf("应透传 402，实际 %d", w2.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// 流式空闲时补 SSE 注释心跳：上游长时间不出字（长思考）不能被反代空闲超时掐掉。
|
||||
func TestStreamHeartbeatWhileIdle(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -542,7 +625,7 @@ func TestCacheWritePriceBilledSeparately(t *testing.T) {
|
||||
defer srv.Close()
|
||||
|
||||
reqBody := `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`
|
||||
costOf := func(h *harness) float64 {
|
||||
costOf := func(h *harness) decimal.Decimal {
|
||||
if w := h.run(t, nil, reqBody); w.Code != http.StatusOK {
|
||||
t.Fatalf("code=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -560,7 +643,7 @@ func TestCacheWritePriceBilledSeparately(t *testing.T) {
|
||||
// 默认未配置：回退输入价 0.15 × 1.25 = 0.1875，加 in/out 的零头
|
||||
h1 := newHarness(t, srv.URL, 0, true, nil)
|
||||
fallback := costOf(h1)
|
||||
if fallback < 0.18 || fallback > 0.20 {
|
||||
if fallback.LessThan(decimal.NewFromFloat(0.18)) || fallback.GreaterThan(decimal.NewFromFloat(0.20)) {
|
||||
t.Fatalf("未配置缓存写价应按输入价×1.25，实际 cost=%v", fallback)
|
||||
}
|
||||
|
||||
@@ -571,10 +654,10 @@ func TestCacheWritePriceBilledSeparately(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
explicit := costOf(h2)
|
||||
if explicit < 1.99 || explicit > 2.02 {
|
||||
if explicit.LessThan(decimal.NewFromFloat(1.99)) || explicit.GreaterThan(decimal.NewFromFloat(2.02)) {
|
||||
t.Fatalf("按 cache_write_price=2.0 计费应为 ~2.0，实际 %v", explicit)
|
||||
}
|
||||
if !(explicit > fallback) {
|
||||
if !explicit.GreaterThan(fallback) {
|
||||
t.Fatalf("显式价应高于回退价：%v vs %v", explicit, fallback)
|
||||
}
|
||||
}
|
||||
@@ -601,7 +684,7 @@ func TestAllChannelsFailedAttributesLastChannel(t *testing.T) {
|
||||
if l.ChannelID == 0 {
|
||||
t.Fatalf("失败记录没归到渠道上：%+v", l)
|
||||
}
|
||||
if l.Status != store.UsageStatusError || l.Cost != 0 {
|
||||
if l.Status != store.UsageStatusError || !l.Cost.IsZero() {
|
||||
t.Fatalf("失败不应产生扣费：%+v", l)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user