feat: 运行期渠道冷却可见并可手动恢复
运行期冷却只存在网关进程内存里，DB 的 health_status 看不见它：渠道被踢到 30s 冷却后，管理后台仍显示 healthy，模型也照常出现在 /v1/models，但请求 返回 503 no_channel，管理员只能去翻日志。 - channel.Service 增 RuntimeState（连续失败次数 + 冷却剩余）与 ClearRuntime； Gateway 包装出 ChannelRuntime / RecoverChannel 供管理端调用。 - GET /api/v1/admin/channels 每项返回 runtime_fails 与 cooldown_seconds。 - 新增 POST /api/v1/admin/channels/:id/recover：清运行期冷却与失败计数，并把 探测写的 health_status 置回 healthy；不改 enabled，上游若真挂着，下一次探测 或再连续 3 次请求失败会重新把它踢掉。 - 渠道页健康列下方显示「冷却中 Ns」/「连续失败 n/3」，仅在冷却或探测不健康时 出现「立即恢复」按钮（卡片与桌面表格两套布局都覆盖）。 仍未处理：冷却到期是裸恢复、没有 half-open 试探；计数是进程内的，多实例各自 为战，要共享得等 Redis 那一步。
This commit is contained in:
Sakurasan committed
1 parent
e865f69a41
commit
d4c5263f1e
8 files changed
+166
-2
No files matched your search
@@ -20,7 +20,7 @@
|
||||
- **用户体系**：注册（开放/邀请码可切换，管理后台可改）、登录（JWT access + HttpOnly refresh cookie）、argon2id 密码
|
||||
- **API Key**：`sk-ot-` 48 位 base62，仅存 SHA-256 哈希，明文一次性展示；支持限额/过期/白名单字段
|
||||
- **用量计费**：请求级 `usage_logs` 异步批量落库，按模型价格扣减余额，日粒度预聚合（`usage_daily`）
|
||||
- **管理 API**：渠道 CRUD + 连通测试 + 模型导入、模型管理 + 定价 + 渠道绑定、用户管理、全局用量/统计、系统配置
|
||||
- **管理 API**：渠道 CRUD + 连通测试 + 立即恢复（清运行期冷却）+ 模型导入、模型管理 + 定价 + 渠道绑定、用户管理、全局用量/统计、系统配置
|
||||
- **前端**（Vue3 + Tailwind，taste-skill 设计，深色优先）
|
||||
- Landing / 登录 / 注册
|
||||
- 控制台：Dashboard（余额/用量/趋势图）、API Keys、用量明细
|
||||
|
||||
@@ -30,18 +30,30 @@ func (h *Handler) AdminChannels(c *gin.Context) {
|
||||
} else if err == nil {
|
||||
masked = "****"
|
||||
}
|
||||
fails, cooldownSec := h.channelRuntime(ch.ID)
|
||||
out = append(out, gin.H{
|
||||
"id": ch.ID, "name": ch.Name, "provider": ch.Provider, "formats": ch.FormatsEffective(),
|
||||
"base_url": ch.BaseURL, "base_urls": ch.BaseURLs,
|
||||
"api_key_masked": masked, "weight": ch.Weight, "priority": ch.Priority,
|
||||
"timeout_ms": ch.TimeoutMS, "max_concurrency": ch.MaxConcurrency,
|
||||
"health_status": ch.HealthStatus, "enabled": ch.Enabled,
|
||||
"runtime_fails": fails, "cooldown_seconds": cooldownSec,
|
||||
"created_at": ch.CreatedAt,
|
||||
})
|
||||
}
|
||||
resp.OK(c, gin.H{"items": out})
|
||||
}
|
||||
|
||||
// channelRuntime 渠道运行期健康：连续失败次数与冷却剩余秒数（0 表示无冷却）。
|
||||
// 运行期冷却只存在于进程内存，DB 的 health_status 看不到它，故列表单独返回。
|
||||
func (h *Handler) channelRuntime(id uint64) (fails int, cooldownSeconds int64) {
|
||||
if h.gw == nil {
|
||||
return 0, 0
|
||||
}
|
||||
fails, remaining := h.gw.ChannelRuntime(id)
|
||||
return fails, int64((remaining + time.Second - time.Nanosecond) / time.Second) // 向上取整
|
||||
}
|
||||
|
||||
type channelBody struct {
|
||||
Name string `json:"name" binding:"required,min=1,max=64"`
|
||||
Provider string `json:"provider"` // 可选：为空时按 formats 推断（兼容旧数据）
|
||||
@@ -369,6 +381,32 @@ func (h *Handler) AdminTestChannel(c *gin.Context) {
|
||||
resp.OK(c, gin.H{"ok": true, "latency_ms": latency, "message": msg})
|
||||
}
|
||||
|
||||
// AdminRecoverChannel POST /api/v1/admin/channels/:id/recover — 立即恢复渠道：
|
||||
// 清掉运行期冷却与连续失败计数，并把探测写的 health_status 置回 healthy。
|
||||
// 不改 enabled；上游若真挂着，下一次探测或连续 3 次请求失败会重新把它踢掉。
|
||||
func (h *Handler) AdminRecoverChannel(c *gin.Context) {
|
||||
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
||||
if err != nil {
|
||||
resp.Fail(c, http.StatusBadRequest, "invalid channel id")
|
||||
return
|
||||
}
|
||||
var ch store.Channel
|
||||
if err := h.a.DB.First(&ch, id).Error; err != nil {
|
||||
resp.Fail(c, http.StatusNotFound, "channel not found")
|
||||
return
|
||||
}
|
||||
if h.gw != nil {
|
||||
h.gw.RecoverChannel(ch.ID)
|
||||
}
|
||||
if err := h.a.DB.Model(&store.Channel{}).Where("id = ?", ch.ID).
|
||||
Update("health_status", store.ChannelHealthHealthy).Error; err != nil {
|
||||
resp.Fail(c, http.StatusInternalServerError, "failed to update channel")
|
||||
return
|
||||
}
|
||||
resp.OK(c, gin.H{"id": ch.ID, "name": ch.Name,
|
||||
"health_status": store.ChannelHealthHealthy, "runtime_fails": 0, "cooldown_seconds": 0})
|
||||
}
|
||||
|
||||
// maskAPIKey 掩码渠道密钥：保留前 7 位与后 4 位，中间固定 ****** 遮蔽。
|
||||
// 示例：xxxxxxx******Mq4Y；密钥较短时退化为仅保留后 4 位。
|
||||
func maskAPIKey(key string) string {
|
||||
|
||||
@@ -111,6 +111,7 @@ func NewRouter(a *app.App, gw *proxy.Gateway) *gin.Engine {
|
||||
admin.PUT("/channels/:id", h.AdminUpdateChannel)
|
||||
admin.DELETE("/channels/:id", h.AdminDeleteChannel)
|
||||
admin.POST("/channels/:id/test", h.AdminTestChannel)
|
||||
admin.POST("/channels/:id/recover", h.AdminRecoverChannel)
|
||||
admin.GET("/channels/:id/models/remote", h.AdminChannelRemoteModels)
|
||||
admin.GET("/channels/:id/models", h.AdminChannelModels)
|
||||
admin.POST("/channels/:id/models", h.AdminChannelAddModel)
|
||||
|
||||
@@ -215,6 +215,32 @@ func (s *Service) ReportSuccess(ch *store.Channel) {
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// RuntimeState 渠道运行期健康快照：连续失败次数与冷却剩余时间（管理后台展示用）。
|
||||
// 只读，不做过期清理——清理仍由 InCooldown 在选路时惰性完成。
|
||||
func (s *Service) RuntimeState(id uint64) (fails int, remaining time.Duration) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
st, ok := s.runt[id]
|
||||
if !ok {
|
||||
return 0, 0
|
||||
}
|
||||
if !st.until.IsZero() {
|
||||
if d := time.Until(st.until); d > 0 {
|
||||
return 0, d // 冷却中：计数在写入时已归零
|
||||
}
|
||||
return 0, 0 // 已到期，等下次选路清理
|
||||
}
|
||||
return st.fails, 0
|
||||
}
|
||||
|
||||
// ClearRuntime 清除运行期状态（管理后台"立即恢复"）。
|
||||
// 不改 DB 的 health_status——那条由探测/管理端点负责，两者语义不同不要混在一起。
|
||||
func (s *Service) ClearRuntime(id uint64) {
|
||||
s.mu.Lock()
|
||||
delete(s.runt, id)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// InCooldown 渠道是否处于运行期冷却；过期则清掉状态、恢复可用。
|
||||
// until 为零值表示只在累计失败、尚未触发冷却，此时不能清状态（否则计数永远攒不满阈值）。
|
||||
func (s *Service) InCooldown(id uint64) bool {
|
||||
|
||||
@@ -205,3 +205,40 @@ func TestRuntimeCooldownThresholdAndExpiry(t *testing.T) {
|
||||
t.Fatal("非连续失败不应触发冷却")
|
||||
}
|
||||
}
|
||||
|
||||
// 管理后台可见性：RuntimeState 报累计失败与冷却剩余，ClearRuntime 立即清零。
|
||||
func TestRuntimeStateAndClear(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
id := seedChannel(t, db, "s", 1, 0, 4)
|
||||
var ch store.Channel
|
||||
db.First(&ch, id)
|
||||
|
||||
s := NewService(db, crypto.NewEncryptor("test-master-key-1234567890"))
|
||||
if fails, rem := s.RuntimeState(id); fails != 0 || rem != 0 {
|
||||
t.Fatalf("初始状态 = %d/%s，期望 0/0", fails, rem)
|
||||
}
|
||||
s.ReportFailure(&ch)
|
||||
s.ReportFailure(&ch)
|
||||
if fails, rem := s.RuntimeState(id); fails != 2 || rem != 0 {
|
||||
t.Fatalf("两次失败后 = %d/%s，期望 2/0", fails, rem)
|
||||
}
|
||||
s.ReportFailure(&ch)
|
||||
fails, rem := s.RuntimeState(id)
|
||||
if fails != 0 || rem <= 0 || rem > s.cooldownFor {
|
||||
t.Fatalf("冷却中 = %d/%s，期望 0/(0,%s]", fails, rem, s.cooldownFor)
|
||||
}
|
||||
if got := len(s.Candidates("")); got != 0 {
|
||||
t.Fatalf("冷却渠道不应进候选，实际 %d", got)
|
||||
}
|
||||
|
||||
s.ClearRuntime(id)
|
||||
if fails, rem := s.RuntimeState(id); fails != 0 || rem != 0 {
|
||||
t.Fatalf("清除后 = %d/%s，期望 0/0", fails, rem)
|
||||
}
|
||||
if s.InCooldown(id) {
|
||||
t.Fatal("清除后不应仍在冷却")
|
||||
}
|
||||
if got := len(s.Candidates("")); got != 1 {
|
||||
t.Fatalf("恢复后应重新进候选，实际 %d", got)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,16 @@ func (g *Gateway) ResetModelPolicy() {
|
||||
g.policyMu.Unlock()
|
||||
}
|
||||
|
||||
// ChannelRuntime 渠道运行期健康快照：连续失败次数 + 冷却剩余时间（管理后台展示）。
|
||||
func (g *Gateway) ChannelRuntime(id uint64) (fails int, remaining time.Duration) {
|
||||
return g.ch.RuntimeState(id)
|
||||
}
|
||||
|
||||
// RecoverChannel 立即清除渠道的运行期冷却与失败计数（管理后台手动恢复）。
|
||||
func (g *Gateway) RecoverChannel(id uint64) {
|
||||
g.ch.ClearRuntime(id)
|
||||
}
|
||||
|
||||
// globalModelRestrictions 读取全局模型允许/禁止列表（缓存 modelPolicyTTL）。
|
||||
func (g *Gateway) globalModelRestrictions() (allow, deny []string) {
|
||||
g.policyMu.Lock()
|
||||
|
||||
@@ -36,6 +36,8 @@ export interface Channel {
|
||||
timeout_ms: number
|
||||
max_concurrency: number
|
||||
health_status: string
|
||||
runtime_fails: number // 运行期连续失败次数（进程内，满 3 次进冷却）
|
||||
cooldown_seconds: number // 运行期冷却剩余秒数，0 表示未冷却
|
||||
enabled: boolean
|
||||
created_at: string
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, reactive, ref } from 'vue'
|
||||
import { PhCaretDown, PhPulse, PhNotePencil, PhTrash, PhStack } from '@phosphor-icons/vue'
|
||||
import { PhCaretDown, PhPulse, PhNotePencil, PhTrash, PhStack, PhArrowsClockwise } from '@phosphor-icons/vue'
|
||||
import { http, errMsg } from '@/api/client'
|
||||
import { useToastStore } from '@/stores/toast'
|
||||
import { PROTOCOL_OPTIONS, protocolShort } from '@/lib/protocol'
|
||||
@@ -127,6 +127,30 @@ async function testChannel(ch: Channel) {
|
||||
}
|
||||
}
|
||||
|
||||
// 运行期冷却是网关进程内的状态，探测写的 health_status 看不到它，所以单独提示
|
||||
function runtimeHint(ch: Channel): string {
|
||||
if (ch.cooldown_seconds > 0) return `冷却中 ${ch.cooldown_seconds}s`
|
||||
if (ch.runtime_fails > 0) return `连续失败 ${ch.runtime_fails}/3`
|
||||
return ''
|
||||
}
|
||||
|
||||
function needRecover(ch: Channel): boolean {
|
||||
return ch.cooldown_seconds > 0 || ch.health_status !== 'healthy'
|
||||
}
|
||||
|
||||
async function recoverChannel(ch: Channel) {
|
||||
busyId.value = ch.id
|
||||
try {
|
||||
await http.post(`/admin/channels/${ch.id}/recover`)
|
||||
toast.ok(`渠道 ${ch.name} 已恢复`)
|
||||
} catch (e) {
|
||||
toast.err(errMsg(e))
|
||||
} finally {
|
||||
busyId.value = null
|
||||
await load()
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
@@ -159,6 +183,9 @@ onMounted(load)
|
||||
{{ ch.health_status }}
|
||||
</Badge>
|
||||
<Badge :variant="ch.enabled ? 'ok' : 'neutral'">{{ ch.enabled ? '启用' : '停用' }}</Badge>
|
||||
<Badge v-if="runtimeHint(ch)" :variant="ch.cooldown_seconds > 0 ? 'err' : 'warn'">
|
||||
{{ runtimeHint(ch) }}
|
||||
</Badge>
|
||||
</div>
|
||||
</div>
|
||||
<p class="mt-2 truncate font-mono text-[11px] text-muted">{{ ch.base_url }}</p>
|
||||
@@ -167,6 +194,15 @@ onMounted(load)
|
||||
<PhPulse :size="13" />
|
||||
{{ busyId === ch.id ? '测试中…' : '测试' }}
|
||||
</button>
|
||||
<button
|
||||
v-if="needRecover(ch)"
|
||||
class="inline-flex items-center gap-1 text-xs text-accent hover:text-accent-strong"
|
||||
:disabled="busyId === ch.id"
|
||||
@click="recoverChannel(ch)"
|
||||
>
|
||||
<PhArrowsClockwise :size="13" />
|
||||
立即恢复
|
||||
</button>
|
||||
<button class="inline-flex items-center gap-1 text-xs text-accent hover:text-accent-strong" @click="toggleDrawer(ch)">
|
||||
<PhStack :size="13" />
|
||||
支持的模型 {{ expandedId === ch.id ? '▴' : '▾' }}
|
||||
@@ -228,6 +264,11 @@ onMounted(load)
|
||||
<Badge :variant="ch.health_status === 'healthy' ? 'ok' : ch.health_status === 'cooldown' ? 'err' : 'warn'">
|
||||
{{ ch.health_status }}
|
||||
</Badge>
|
||||
<div
|
||||
v-if="runtimeHint(ch)"
|
||||
class="mt-1 font-mono text-[11px]"
|
||||
:class="ch.cooldown_seconds > 0 ? 'text-err' : 'text-warn'"
|
||||
>{{ runtimeHint(ch) }}</div>
|
||||
</td>
|
||||
<td class="px-4 py-2.5 text-xs text-muted">{{ ch.enabled ? '是' : '否' }}</td>
|
||||
<td class="px-4 py-2.5 text-right">
|
||||
@@ -236,6 +277,15 @@ onMounted(load)
|
||||
<PhPulse :size="13" />
|
||||
{{ busyId === ch.id ? '测试中…' : '测试' }}
|
||||
</button>
|
||||
<button
|
||||
v-if="needRecover(ch)"
|
||||
class="inline-flex items-center gap-1 text-xs text-accent hover:text-accent-strong"
|
||||
:disabled="busyId === ch.id"
|
||||
@click="recoverChannel(ch)"
|
||||
>
|
||||
<PhArrowsClockwise :size="13" />
|
||||
立即恢复
|
||||
</button>
|
||||
<button class="inline-flex items-center gap-1 text-xs text-muted hover:text-ink" @click="openEdit(ch)">
|
||||
<PhNotePencil :size="13" />
|
||||
编辑
|
||||
|
||||
Reference in new issue
Block a user