perf: 流式错误判定预过滤、再减一个写索引,加可选明细保留期

- 流式放行前的上游错误判定加廉价子串预过滤("error" / response.failed):
  长思考阶段几十上百条事件不再逐条 json.Unmarshal。
- usage_logs 去掉 idx_channel_created:没有任何查询按 channel_id 过滤该表
  (渠道统计走 created_at 时间窗后在内存分组),复合索引只是每行一次多余的
  B-tree 插入;schema v9 迁移里显式 DROP 旧库索引。至此该表只剩
  (user_id, created_at)、created_at 两个真被用到的索引。
- 新增可选保留期(OT_USAGE_RETENTION_DAYS,默认 0=永久保留):每天分批
  (2000 行/批、批间 50ms)清理过期 usage_logs,避免长期运行后时间窗查询
  与备份随表增长变慢;余额流水 balance_logs 不在清理范围。

参数复测(记账稳态吞吐,3 次取中位):
- batch 64(当前)30.2k rows/s;batch 128 19.0k、batch 256 14.3k(更差,保持 64)
- wal_autocheckpoint(4000)、cache_size(-64MB) 与当前配置无差别,不引入

验证:go vet、go test ./...、-race(usage/proxy/store)全绿;新增保留期三项测试
(只删明细不动流水、未开启不动数据、跨分批删完)。
This commit is contained in:
Sakurasan committed 2026-10-30 16:45:57 +08:00
1 parent 8e5fe16611
commit 531a03c836
9 files changed
+224 -20

No files matched your search

+84
View File
@@ -0,0 +1,84 @@
package usage
import (
"context"
"fmt"
"testing"
"time"
"github.com/openteam/server/internal/store"
)
// 保留期只删过期明细,余额流水一条不动;未开启(0)时什么都不做。
func TestRetentionPurgesOldLogsOnly(t *testing.T) {
db := openTestDB(t)
u := seedUser(t, db, 100)
old := time.Now().UTC().AddDate(0, 0, -40)
fresh := time.Now().UTC()
for i := 0; i < 5; i++ {
db.Create(&store.UsageLog{TraceID: fmt.Sprintf("old-%d", i), UserID: u, ModelName: "m1",
Cost: 0.01, Status: store.UsageStatusSuccess, CreatedAt: old})
}
for i := 0; i < 3; i++ {
db.Create(&store.UsageLog{TraceID: fmt.Sprintf("new-%d", i), UserID: u, ModelName: "m1",
Cost: 0.01, Status: store.UsageStatusSuccess, CreatedAt: fresh})
}
db.Create(&store.BalanceLog{UserID: u, Change: -0.05, BalanceAfter: 99.95,
Type: store.BalanceTypeUsage, RefID: "usage:old-0"})
n := purgeOlderThan(context.Background(), db, 30)
if n != 5 {
t.Fatalf("应删除 5 条过期明细,实际 %d", n)
}
var logs, ledgers int64
db.Model(&store.UsageLog{}).Count(&logs)
db.Model(&store.BalanceLog{}).Count(&ledgers)
if logs != 3 {
t.Fatalf("应保留 3 条新明细,实际 %d", logs)
}
if ledgers != 1 {
t.Fatalf("余额流水不应被清理,实际 %d 条", ledgers)
}
}
// 未配置保留期时不启动、不删数据。
func TestRetentionDisabled(t *testing.T) {
db := openTestDB(t)
u := seedUser(t, db, 100)
db.Create(&store.UsageLog{TraceID: "x", UserID: u, ModelName: "m1",
Cost: 0.01, Status: store.UsageStatusSuccess,
CreatedAt: time.Now().UTC().AddDate(0, 0, -400)})
stop := StartRetention(context.Background(), db, 0)
stop()
var n int64
db.Model(&store.UsageLog{}).Count(&n)
if n != 1 {
t.Fatalf("未开启保留期时不应删除数据,实际剩 %d", n)
}
}
// 分批删除:行数超过一批时应全部删完。
func TestRetentionBatches(t *testing.T) {
db := openTestDB(t)
u := seedUser(t, db, 100)
old := time.Now().UTC().AddDate(0, 0, -90)
logs := make([]store.UsageLog, 0, retentionBatch+50)
for i := 0; i < retentionBatch+50; i++ {
logs = append(logs, store.UsageLog{TraceID: fmt.Sprintf("b-%d", i), UserID: u,
ModelName: "m1", Cost: 0.001, Status: store.UsageStatusSuccess, CreatedAt: old})
}
if err := db.CreateInBatches(logs, 500).Error; err != nil {
t.Fatal(err)
}
n := purgeOlderThan(context.Background(), db, 30)
if n != int64(len(logs)) {
t.Fatalf("应删除 %d 条,实际 %d", len(logs), n)
}
var left int64
db.Model(&store.UsageLog{}).Count(&left)
if left != 0 {
t.Fatalf("应全部删完,实际剩 %d", left)
}
}