feat: 金额 decimal 精确计费、failover 按失败类型封顶、渠道小时桶预聚合、分层守护测试

- 计费:19 个金额字段 float64→decimal.Decimal,DB 列保持 numeric(20,8) 零迁移;
  API 金额仍为 JSON number(MarshalJSONWithoutQuotes),前端零改动
- failover:保留时间预算,新增 OT_PROXY_FAILOVER_CAPS_NET/SERVER/RATE/QUOTA
  按失败类型封顶换渠道次数(429/配额类瞬时失败时间预算形同虚设)
- 预聚合:新增 usage_channel_hourlies(schema v10),记账事务内同步 upsert,
  长窗口渠道统计走桶不再扫明细,spent_last_30d 改扫 usage_daily,v10 一次性回填幂等
- 守护:internal/arch_test.go 用 go/parser 强制分层依赖(fail-closed)
This commit is contained in:
Sakurasan committed 2026-11-09 00:41:48 +08:00
1 parent 5d6e5bdb04
commit b2e500ff51
25 files changed
+841 -253

No files matched your search

+6 -5
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/openteam/server/internal/store"
"github.com/shopspring/decimal"
)
// 保留期只删过期明细,余额流水一条不动;未开启(0)时什么都不做。
@@ -18,13 +19,13 @@ func TestRetentionPurgesOldLogsOnly(t *testing.T) {
fresh := time.Now().UTC()
for i := 0; i < 5; i++ {
db.Create(&store.UsageLog{TraceID: fmt.Sprintf("old-%d", i), UserID: u, ModelName: "m1",
Cost: 0.01, Status: store.UsageStatusSuccess, CreatedAt: old})
Cost: decimal.NewFromFloat(0.01), Status: store.UsageStatusSuccess, CreatedAt: old})
}
for i := 0; i < 3; i++ {
db.Create(&store.UsageLog{TraceID: fmt.Sprintf("new-%d", i), UserID: u, ModelName: "m1",
Cost: 0.01, Status: store.UsageStatusSuccess, CreatedAt: fresh})
Cost: decimal.NewFromFloat(0.01), Status: store.UsageStatusSuccess, CreatedAt: fresh})
}
db.Create(&store.BalanceLog{UserID: u, Change: -0.05, BalanceAfter: 99.95,
db.Create(&store.BalanceLog{UserID: u, Change: decimal.NewFromFloat(-0.05), BalanceAfter: decimal.NewFromFloat(99.95),
Type: store.BalanceTypeUsage, RefID: "usage:old-0"})
n := purgeOlderThan(context.Background(), db, 30)
@@ -47,7 +48,7 @@ func TestRetentionDisabled(t *testing.T) {
db := openTestDB(t)
u := seedUser(t, db, 100)
db.Create(&store.UsageLog{TraceID: "x", UserID: u, ModelName: "m1",
Cost: 0.01, Status: store.UsageStatusSuccess,
Cost: decimal.NewFromFloat(0.01), Status: store.UsageStatusSuccess,
CreatedAt: time.Now().UTC().AddDate(0, 0, -400)})
stop := StartRetention(context.Background(), db, 0)
@@ -67,7 +68,7 @@ func TestRetentionBatches(t *testing.T) {
logs := make([]store.UsageLog, 0, retentionBatch+50)
for i := 0; i < retentionBatch+50; i++ {
logs = append(logs, store.UsageLog{TraceID: fmt.Sprintf("b-%d", i), UserID: u,
ModelName: "m1", Cost: 0.001, Status: store.UsageStatusSuccess, CreatedAt: old})
ModelName: "m1", Cost: decimal.NewFromFloat(0.001), Status: store.UsageStatusSuccess, CreatedAt: old})
}
if err := db.CreateInBatches(logs, 500).Error; err != nil {
t.Fatal(err)
+73 -6
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/openteam/server/internal/store"
"github.com/shopspring/decimal"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
@@ -207,7 +208,7 @@ func (r *Recorder) bill(tx *gorm.DB, logs []*store.UsageLog) error {
billable := make([]*store.UsageLog, 0, len(logs))
refs := make([]string, 0, len(logs))
for _, l := range logs {
if (l.Status != store.UsageStatusSuccess && l.Status != store.UsageStatusCanceled) || l.Cost <= 0 {
if (l.Status != store.UsageStatusSuccess && l.Status != store.UsageStatusCanceled) || !l.Cost.IsPositive() {
continue
}
billable = append(billable, l)
@@ -250,9 +251,9 @@ func (r *Recorder) bill(tx *gorm.DB, logs []*store.UsageLog) error {
ledger := make([]store.BalanceLog, 0, len(billable))
for _, uid := range order {
rows := byUser[uid]
total := 0.0
total := decimal.Zero
for _, l := range rows {
total += l.Cost
total = total.Add(l.Cost)
}
var user store.User
if err := tx.First(&user, uid).Error; err != nil {
@@ -270,10 +271,10 @@ func (r *Recorder) bill(tx *gorm.DB, logs []*store.UsageLog) error {
// 流水里的 balance_after 是批内累计值,与逐行扣款时的语义一致
running := user.Balance
for _, l := range rows {
running -= l.Cost
running = running.Sub(l.Cost)
ledger = append(ledger, store.BalanceLog{
UserID: uid,
Change: -l.Cost,
Change: l.Cost.Neg(),
BalanceAfter: running,
Type: store.BalanceTypeUsage,
RefID: usageRefID(l.TraceID),
@@ -312,7 +313,7 @@ func (r *Recorder) bill(tx *gorm.DB, logs []*store.UsageLog) error {
d.InputTokens += l.InputTokens
d.OutputTokens += l.OutputTokens
d.CacheReadTokens += l.CacheReadTokens
d.Cost += l.Cost
d.Cost = d.Cost.Add(l.Cost)
}
for _, k := range dailyOrder {
d := agg[k]
@@ -329,6 +330,72 @@ func (r *Recorder) bill(tx *gorm.DB, logs []*store.UsageLog) error {
return err
}
}
// 6. 渠道×小时桶:与日聚合同事务 upsert。长窗口(6h/24h/7d)的渠道统计从桶里
// 读取,不再扫 usage_logs(明细有保留期,桶是完整历史)。计费口径与日聚合一致:
// token/成本只累计可计费且未重复的行;状态计数(含 error)计全部行。
type chKey struct {
ch uint64
hour string
}
chAgg := map[chKey]*store.UsageChannelHourly{}
chOrder := make([]chKey, 0, len(logs))
for _, l := range logs {
if l.ChannelID == 0 {
continue
}
k := chKey{ch: l.ChannelID, hour: l.CreatedAt.UTC().Format("2006-01-02T15")}
b, ok := chAgg[k]
if !ok {
b = &store.UsageChannelHourly{ChannelID: k.ch, Hour: k.hour}
chAgg[k] = b
chOrder = append(chOrder, k)
}
b.Requests++
switch l.Status {
case store.UsageStatusSuccess:
b.Success++
b.LatencySum += int64(l.LatencyMS)
if l.LatencyMS > b.LatencyMax {
b.LatencyMax = l.LatencyMS
}
case store.UsageStatusCanceled:
b.Canceled++
default:
b.Errors++
}
if l.Cost.IsPositive() && seen[usageRefID(l.TraceID)] {
b.InputTokens += l.InputTokens
b.OutputTokens += l.OutputTokens
b.CacheReadTokens += l.CacheReadTokens
b.Cost = b.Cost.Add(l.Cost)
}
}
for _, k := range chOrder {
b := chAgg[k]
// SQLite 的双参 MAX 是标量取大,PG 用 GREATEST
latMax := gorm.Expr("MAX(latency_max, ?)", b.LatencyMax)
if tx.Dialector != nil && tx.Dialector.Name() == "postgres" {
latMax = gorm.Expr("GREATEST(latency_max, ?)", b.LatencyMax)
}
if err := tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "channel_id"}, {Name: "hour"}},
DoUpdates: clause.Assignments(map[string]any{
"requests": gorm.Expr("requests + ?", b.Requests),
"success": gorm.Expr("success + ?", b.Success),
"canceled": gorm.Expr("canceled + ?", b.Canceled),
"errors": gorm.Expr("errors + ?", b.Errors),
"input_tokens": gorm.Expr("input_tokens + ?", b.InputTokens),
"output_tokens": gorm.Expr("output_tokens + ?", b.OutputTokens),
"cache_read_tokens": gorm.Expr("cache_read_tokens + ?", b.CacheReadTokens),
"cost": gorm.Expr("cost + ?", b.Cost),
"latency_sum": gorm.Expr("latency_sum + ?", b.LatencySum),
"latency_max": latMax,
}),
}).Create(b).Error; err != nil {
return err
}
}
return nil
}
+68 -18
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/openteam/server/internal/store"
"github.com/shopspring/decimal"
"gorm.io/gorm"
)
@@ -75,11 +76,13 @@ func TestBillingDeductsBalanceAtomically(t *testing.T) {
m := seedModel(t, db)
rec := NewRecorder(db)
cost := decimal.NewFromFloat(0.15).Add(decimal.NewFromFloat(0.60)).
Mul(decimal.NewFromInt(1000)).Div(decimal.NewFromInt(1e6)).Round(8)
rec.Record(&store.UsageLog{
TraceID: "bill", UserID: u, ModelID: m, ModelName: "m1",
InputTokens: 1000, OutputTokens: 1000,
InputPrice: 0.15, OutputPrice: 0.60,
Cost: 0.15*1000/1e6 + 0.60*1000/1e6,
InputPrice: decimal.NewFromFloat(0.15), OutputPrice: decimal.NewFromFloat(0.60),
Cost: cost,
Status: store.UsageStatusSuccess,
LatencyMS: 5,
CreatedAt: time.Now().UTC(),
@@ -90,15 +93,15 @@ func TestBillingDeductsBalanceAtomically(t *testing.T) {
if err := db.First(&user, u).Error; err != nil {
t.Fatal(err)
}
want := 1 - (0.15*1000/1e6 + 0.60*1000/1e6)
if diff := user.Balance - want; diff > 1e-9 || diff < -1e-9 {
t.Fatalf("余额应扣到 %.10f,实际 %.10f", want, user.Balance)
want := decimal.NewFromInt(1).Sub(cost)
if !user.Balance.Equal(want) {
t.Fatalf("余额应精确扣到 %s,实际 %s", want, user.Balance)
}
var bl store.BalanceLog
if err := db.Where("user_id = ?", u).First(&bl).Error; err != nil {
t.Fatalf("余额流水应落库: %v", err)
}
if bl.Change >= 0 {
if !bl.Change.IsNegative() {
t.Fatalf("流水应为负数扣费,实际 %v", bl.Change)
}
}
@@ -124,7 +127,7 @@ func openTestDB(t *testing.T) *gorm.DB {
func seedUser(t *testing.T, db *gorm.DB, balance float64) uint64 {
t.Helper()
u := store.User{Username: "u" + fmt.Sprint(time.Now().UnixNano()), Email: "u@x.com",
PasswordHash: "x", Role: store.RoleUser, Balance: balance, Status: store.UserStatusActive}
PasswordHash: "x", Role: store.RoleUser, Balance: decimal.NewFromFloat(balance), Status: store.UserStatusActive}
if err := db.Create(&u).Error; err != nil {
t.Fatal(err)
}
@@ -133,7 +136,7 @@ func seedUser(t *testing.T, db *gorm.DB, balance float64) uint64 {
func seedModel(t *testing.T, db *gorm.DB) uint64 {
t.Helper()
m := store.Model{Name: "m1", InputPrice: 0.15, OutputPrice: 0.60, Enabled: true}
m := store.Model{Name: "m1", InputPrice: decimal.NewFromFloat(0.15), OutputPrice: decimal.NewFromFloat(0.60), Enabled: true}
if err := db.Create(&m).Error; err != nil {
t.Fatal(err)
}
@@ -151,7 +154,7 @@ func TestBillingRefIDImpotency(t *testing.T) {
rec := NewRecorder(db)
mk := func(trace string) *store.UsageLog {
return &store.UsageLog{TraceID: trace, UserID: u, ModelName: "m1",
Status: store.UsageStatusSuccess, Cost: 1, CreatedAt: time.Now().UTC()}
Status: store.UsageStatusSuccess, Cost: decimal.NewFromInt(1), CreatedAt: time.Now().UTC()}
}
rec.Record(mk("dup"))
rec.Record(mk("dup"))
@@ -161,7 +164,7 @@ func TestBillingRefIDImpotency(t *testing.T) {
if err := db.First(&user, u).Error; err != nil {
t.Fatal(err)
}
if user.Balance != 4 {
if !user.Balance.Equal(decimal.NewFromInt(4)) {
t.Errorf("balance = %v, want 4 (重复 trace 不重复扣款)", user.Balance)
}
var n int64
@@ -182,7 +185,7 @@ func TestBillingAggregatesBatch(t *testing.T) {
for i := 0; i < n; i++ {
rec.Record(&store.UsageLog{
TraceID: fmt.Sprintf("agg-%d", i), UserID: u, ModelID: m, ModelName: "m1",
InputTokens: 10, OutputTokens: 5, Cost: 0.01,
InputTokens: 10, OutputTokens: 5, Cost: decimal.NewFromFloat(0.01),
Status: store.UsageStatusSuccess, CreatedAt: time.Now().UTC(),
})
}
@@ -192,7 +195,7 @@ func TestBillingAggregatesBatch(t *testing.T) {
if err := db.First(&user, u).Error; err != nil {
t.Fatal(err)
}
if want := 100 - 0.01*n; user.Balance-want > 1e-9 || user.Balance-want < -1e-9 {
if want := decimal.NewFromInt(100).Sub(decimal.NewFromFloat(0.01).Mul(decimal.NewFromInt(n))); !user.Balance.Equal(want) {
t.Fatalf("余额 = %v,期望 %v", user.Balance, want)
}
var ledgers int64
@@ -208,13 +211,13 @@ func TestBillingAggregatesBatch(t *testing.T) {
if daily[0].Requests != n {
t.Fatalf("日聚合 requests = %d,期望 %d", daily[0].Requests, n)
}
if d := daily[0].Cost - 0.01*n; d > 1e-9 || d < -1e-9 {
t.Fatalf("日聚合 cost = %v,期望 %v", daily[0].Cost, 0.01*n)
if want := decimal.NewFromFloat(0.01).Mul(decimal.NewFromInt(n)); !daily[0].Cost.Equal(want) {
t.Fatalf("日聚合 cost = %v,期望 %v", daily[0].Cost, want)
}
// 流水里的 balance_after 应与扣款后余额一致(批内累计)
var last store.BalanceLog
db.Where("user_id = ?", u).Order("id DESC").First(&last)
if last.BalanceAfter-user.Balance > 1e-9 || user.Balance-last.BalanceAfter > 1e-9 {
if !last.BalanceAfter.Equal(user.Balance) {
t.Fatalf("最后一条流水 balance_after = %v,实际余额 %v", last.BalanceAfter, user.Balance)
}
}
@@ -226,7 +229,7 @@ func TestRecorderShardedByUser(t *testing.T) {
var users []uint64
for i := 0; i < 3; i++ {
u := store.User{Username: fmt.Sprintf("shard-u%d", i), Email: fmt.Sprintf("shard%d@x.com", i),
PasswordHash: "x", Role: store.RoleUser, Balance: 100, Status: store.UserStatusActive}
PasswordHash: "x", Role: store.RoleUser, Balance: decimal.NewFromInt(100), Status: store.UserStatusActive}
if err := db.Create(&u).Error; err != nil {
t.Fatal(err)
}
@@ -239,7 +242,7 @@ func TestRecorderShardedByUser(t *testing.T) {
for _, u := range users {
rec.Record(&store.UsageLog{
TraceID: fmt.Sprintf("u%d-%d", u, i), UserID: u, ModelName: "m1",
Cost: 0.001, Status: store.UsageStatusSuccess, CreatedAt: time.Now().UTC(),
Cost: decimal.NewFromFloat(0.001), Status: store.UsageStatusSuccess, CreatedAt: time.Now().UTC(),
})
}
}
@@ -258,7 +261,7 @@ func TestRecorderShardedByUser(t *testing.T) {
t.Fatalf("用户 %d 流水 %d 条,期望 %d", u, len(logs), per)
}
for i := 1; i < len(logs); i++ {
if logs[i].BalanceAfter > logs[i-1].BalanceAfter {
if logs[i].BalanceAfter.GreaterThan(logs[i-1].BalanceAfter) {
t.Fatalf("用户 %d 的流水余额应单调递减(顺序错乱):%v → %v",
u, logs[i-1].BalanceAfter, logs[i].BalanceAfter)
}
@@ -266,6 +269,53 @@ func TestRecorderShardedByUser(t *testing.T) {
}
}
// 渠道×小时桶:状态计数含 error 行,token/成本只累计可计费行,跨小时分桶。
func TestChannelHourlyBucketAggregated(t *testing.T) {
db := openTestDB(t)
u := seedUser(t, db, 100)
m := seedModel(t, db)
rec := NewRecorder(db)
now := time.Now().UTC()
hour := now.Format("2006-01-02T15")
prevHour := now.Add(-time.Hour).Format("2006-01-02T15")
mk := func(trace, status string, at time.Time, ch uint64) *store.UsageLog {
return &store.UsageLog{TraceID: trace, UserID: u, ModelID: m, ModelName: "m1",
ChannelID: ch, InputTokens: 100, OutputTokens: 50, Cost: decimal.NewFromFloat(0.02),
Status: status, LatencyMS: 200, CreatedAt: at}
}
rec.Record(mk("h1", store.UsageStatusSuccess, now, 7))
rec.Record(mk("h2", store.UsageStatusSuccess, now, 7))
rec.Record(mk("h3", store.UsageStatusError, now, 7))
rec.Record(mk("h4", store.UsageStatusSuccess, now.Add(-time.Hour), 7))
rec.Record(mk("h5", store.UsageStatusSuccess, now, 0)) // channel_id=0 不入桶
rec.Close()
var buckets []store.UsageChannelHourly
db.Where("channel_id = ?", 7).Order("hour DESC").Find(&buckets)
if len(buckets) != 2 {
t.Fatalf("应有两个小时桶,实际 %d", len(buckets))
}
cur, prev := buckets[0], buckets[1]
if cur.Hour != hour || prev.Hour != prevHour {
t.Fatalf("桶小时错误:%q / %q,期望 %q / %q", cur.Hour, prev.Hour, hour, prevHour)
}
if cur.Requests != 3 || cur.Success != 2 || cur.Errors != 1 {
t.Fatalf("当前桶计数 = %d/%d/%d,期望 3/2/1(requests/success/errors)", cur.Requests, cur.Success, cur.Errors)
}
// 只有 success 行计延迟
if cur.LatencySum != 400 || cur.LatencyMax != 200 {
t.Fatalf("延迟聚合 = sum %d max %d,期望 400/200", cur.LatencySum, cur.LatencyMax)
}
// token/成本只累计可计费行(2×success)
if cur.InputTokens != 200 || cur.Cost.String() != "0.04" {
t.Fatalf("token/成本 = %d/%s,期望 200/0.04", cur.InputTokens, cur.Cost)
}
if prev.Requests != 1 || prev.Success != 1 {
t.Fatalf("上一小时桶计数 = %d/%d,期望 1/1", prev.Requests, prev.Success)
}
}
// 分片函数:同一用户恒定同片,且覆盖所有分片。
func TestShardOfStable(t *testing.T) {
rec := NewRecorder(openTestDB(t))