feat: 运行期渠道冷却可见并可手动恢复
运行期冷却只存在网关进程内存里,DB 的 health_status 看不见它:渠道被踢到 30s 冷却后,管理后台仍显示 healthy,模型也照常出现在 /v1/models,但请求 返回 503 no_channel,管理员只能去翻日志。 - channel.Service 增 RuntimeState(连续失败次数 + 冷却剩余)与 ClearRuntime; Gateway 包装出 ChannelRuntime / RecoverChannel 供管理端调用。 - GET /api/v1/admin/channels 每项返回 runtime_fails 与 cooldown_seconds。 - 新增 POST /api/v1/admin/channels/:id/recover:清运行期冷却与失败计数,并把 探测写的 health_status 置回 healthy;不改 enabled,上游若真挂着,下一次探测 或再连续 3 次请求失败会重新把它踢掉。 - 渠道页健康列下方显示「冷却中 Ns」/「连续失败 n/3」,仅在冷却或探测不健康时 出现「立即恢复」按钮(卡片与桌面表格两套布局都覆盖)。 仍未处理:冷却到期是裸恢复、没有 half-open 试探;计数是进程内的,多实例各自 为战,要共享得等 Redis 那一步。
This commit is contained in:
1 parent
e865f69a41
commit
d4c5263f1e
8 files changed
+166
-2
No files matched your search
@@ -30,18 +30,30 @@ func (h *Handler) AdminChannels(c *gin.Context) {
|
||||
} else if err == nil {
|
||||
masked = "****"
|
||||
}
|
||||
fails, cooldownSec := h.channelRuntime(ch.ID)
|
||||
out = append(out, gin.H{
|
||||
"id": ch.ID, "name": ch.Name, "provider": ch.Provider, "formats": ch.FormatsEffective(),
|
||||
"base_url": ch.BaseURL, "base_urls": ch.BaseURLs,
|
||||
"api_key_masked": masked, "weight": ch.Weight, "priority": ch.Priority,
|
||||
"timeout_ms": ch.TimeoutMS, "max_concurrency": ch.MaxConcurrency,
|
||||
"health_status": ch.HealthStatus, "enabled": ch.Enabled,
|
||||
"runtime_fails": fails, "cooldown_seconds": cooldownSec,
|
||||
"created_at": ch.CreatedAt,
|
||||
})
|
||||
}
|
||||
resp.OK(c, gin.H{"items": out})
|
||||
}
|
||||
|
||||
// channelRuntime 渠道运行期健康:连续失败次数与冷却剩余秒数(0 表示无冷却)。
|
||||
// 运行期冷却只存在于进程内存,DB 的 health_status 看不到它,故列表单独返回。
|
||||
func (h *Handler) channelRuntime(id uint64) (fails int, cooldownSeconds int64) {
|
||||
if h.gw == nil {
|
||||
return 0, 0
|
||||
}
|
||||
fails, remaining := h.gw.ChannelRuntime(id)
|
||||
return fails, int64((remaining + time.Second - time.Nanosecond) / time.Second) // 向上取整
|
||||
}
|
||||
|
||||
type channelBody struct {
|
||||
Name string `json:"name" binding:"required,min=1,max=64"`
|
||||
Provider string `json:"provider"` // 可选:为空时按 formats 推断(兼容旧数据)
|
||||
@@ -369,6 +381,32 @@ func (h *Handler) AdminTestChannel(c *gin.Context) {
|
||||
resp.OK(c, gin.H{"ok": true, "latency_ms": latency, "message": msg})
|
||||
}
|
||||
|
||||
// AdminRecoverChannel POST /api/v1/admin/channels/:id/recover — 立即恢复渠道:
|
||||
// 清掉运行期冷却与连续失败计数,并把探测写的 health_status 置回 healthy。
|
||||
// 不改 enabled;上游若真挂着,下一次探测或连续 3 次请求失败会重新把它踢掉。
|
||||
func (h *Handler) AdminRecoverChannel(c *gin.Context) {
|
||||
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
||||
if err != nil {
|
||||
resp.Fail(c, http.StatusBadRequest, "invalid channel id")
|
||||
return
|
||||
}
|
||||
var ch store.Channel
|
||||
if err := h.a.DB.First(&ch, id).Error; err != nil {
|
||||
resp.Fail(c, http.StatusNotFound, "channel not found")
|
||||
return
|
||||
}
|
||||
if h.gw != nil {
|
||||
h.gw.RecoverChannel(ch.ID)
|
||||
}
|
||||
if err := h.a.DB.Model(&store.Channel{}).Where("id = ?", ch.ID).
|
||||
Update("health_status", store.ChannelHealthHealthy).Error; err != nil {
|
||||
resp.Fail(c, http.StatusInternalServerError, "failed to update channel")
|
||||
return
|
||||
}
|
||||
resp.OK(c, gin.H{"id": ch.ID, "name": ch.Name,
|
||||
"health_status": store.ChannelHealthHealthy, "runtime_fails": 0, "cooldown_seconds": 0})
|
||||
}
|
||||
|
||||
// maskAPIKey 掩码渠道密钥:保留前 7 位与后 4 位,中间固定 ****** 遮蔽。
|
||||
// 示例:xxxxxxx******Mq4Y;密钥较短时退化为仅保留后 4 位。
|
||||
func maskAPIKey(key string) string {
|
||||
|
||||
Reference in new issue
Block a user