Files
openteam/server/internal/api/admin_channels.go
Sakurasan 979973ddde feat: 流式首包缓冲容错(hold)与失败分类冷却
流式请求在首个内容事件到达前由网关缓冲(借鉴 magpie 的 holdWriter):上游的错误
事件、空流、断连、协议形态不符(要流给了整包)都在放行前换渠道消化,客户端无感知。
扣住期间照常发 SSE 注释心跳;缓冲超过 1MB 直接放行退化为普通转发,判定失效不吞正文。

- 新增 proxy/hold.go:streamHold(缓冲/放行/心跳/首 token 计时与放行互斥)、
  sseEventHasContent(chat/messages/responses 三种协议的内容判定)、
  sseUpstreamError(200+流式形式的错误事件识别)、classifyUpstreamError +
  quotaSignals(余额/配额特征识别)
- 非流式改为先读完整响应再写头:读失败(网络错误/超时)发生在写出之前,可换渠道
  重试;此前先写 200 头再读,超时只能给客户端留一个 200 空响应
- 协议形态校验:要流却给了整包(didNotStream)按失败换渠道,反复如此的渠道会被
  健康计数剔除
- 记账:每次尝试独立 trySink,只有成功走到 EOF 才并入请求级 sink——失败渠道的
  message_start 用量不计入账单
- 冷却按原因分类(channel.FailKind):余额/配额类第一次失败即冷却 30 分钟(429 不
  一定是限流,智谱 429「余额不足」也在此列);其余按连续进冷却次数指数退避
  30s→60s→120s,封顶 10 分钟;短于退避值的 Retry-After 不再缩短冷却
- 新增 OT_PROXY_FIRST_TOKEN_WAIT(默认 0=不限):放行前等内容的最长时间,超时即
  取消本次尝试换下一个渠道
- 渠道列表接口返回 cooldown_reason,后台可区分「限流」与「余额耗尽」
- 测试:hold 缓冲/放行/心跳、内容与错误事件判定、失败分类与指数退避、容量热生效
- docs/magpie-plugin-mechanism.md:参考实现的插件机制调研
2026-10-30 02:43:23 +08:00

432 lines
14 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package api
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/openteam/server/internal/pkg/resp"
"github.com/openteam/server/internal/store"
)
// AdminChannels GET /api/v1/admin/channels — 渠道列表(不返回加密 key,返回掩码)。
func (h *Handler) AdminChannels(c *gin.Context) {
var chs []store.Channel
if err := h.a.DB.Order("id ASC").Find(&chs).Error; err != nil {
resp.Fail(c, http.StatusInternalServerError, "failed to load channels")
return
}
out := make([]gin.H, 0, len(chs))
for _, ch := range chs {
masked := ""
if key, err := h.a.Enc.Decrypt(ch.APIKeyEnc); err == nil && len(key) > 8 {
masked = maskAPIKey(key)
} else if err == nil {
masked = "****"
}
fails, cooldownSec, reason := h.channelRuntime(ch.ID)
out = append(out, gin.H{
"id": ch.ID, "name": ch.Name, "provider": ch.Provider, "formats": ch.FormatsEffective(),
"base_url": ch.BaseURL, "base_urls": ch.BaseURLs,
"api_key_masked": masked, "weight": ch.Weight, "priority": ch.Priority,
"timeout_ms": ch.TimeoutMS, "max_concurrency": ch.MaxConcurrency,
"health_status": ch.HealthStatus, "enabled": ch.Enabled,
"runtime_fails": fails, "cooldown_seconds": cooldownSec, "cooldown_reason": reason,
"created_at": ch.CreatedAt,
})
}
resp.OK(c, gin.H{"items": out})
}
// channelRuntime 渠道运行期健康:连续失败次数、冷却剩余秒数(0 表示无冷却)与最近失败原因。
// 运行期冷却只存在于进程内存,DB 的 health_status 看不到它,故列表单独返回。
func (h *Handler) channelRuntime(id uint64) (fails int, cooldownSeconds int64, reason string) {
if h.gw == nil {
return 0, 0, ""
}
fails, remaining, reason := h.gw.ChannelRuntime(id)
return fails, int64((remaining + time.Second - time.Nanosecond) / time.Second), reason // 向上取整
}
type channelBody struct {
Name string `json:"name" binding:"required,min=1,max=64"`
Provider string `json:"provider"` // 可选:为空时按 formats 推断(兼容旧数据)
Formats []string `json:"formats"` // 原生支持的协议 chat|responses|messages(主配置)
BaseURL string `json:"base_url"` // 可选:留空按供应商默认;支持前缀或完整端点
BaseURLs map[string]string `json:"base_urls"` // 分协议 base_url 覆盖(chat/responses/messages)
APIKey string `json:"api_key"`
Weight *int `json:"weight"`
Priority *int `json:"priority"`
TimeoutMS *int `json:"timeout_ms"`
MaxConcurrency *int `json:"max_concurrency"`
Enabled *bool `json:"enabled"`
}
// normalizeBaseURLs 校验并清理分协议 base_url。
func normalizeBaseURLs(m map[string]string) map[string]string {
if len(m) == 0 {
return nil
}
out := map[string]string{}
for k, v := range m {
if validFormats[k] && strings.TrimSpace(v) != "" {
out[k] = strings.TrimRight(strings.TrimSpace(v), "/")
}
}
if len(out) == 0 {
return nil
}
return out
}
// resolveBaseURL 渠道 base_url:留空按供应商默认;网关按内容智能识别前缀/完整端点。
func resolveBaseURL(provider, raw string) (string, error) {
base := strings.TrimRight(raw, "/")
if base == "" {
switch provider {
case store.ChannelProviderOpenAI:
base = "https://api.openai.com"
case store.ChannelProviderAnthropic:
base = "https://api.anthropic.com"
}
}
if base == "" {
return "", errors.New("base_url required for compatible channels")
}
return base, nil
}
func validateProvider(p string) bool {
return p == store.ChannelProviderOpenAI || p == store.ChannelProviderAnthropic || p == store.ChannelProviderCompatible
}
var validFormats = map[string]bool{
store.FormatChat: true, store.FormatResponses: true, store.FormatMessages: true,
}
// deriveProvider 按格式推断供应商(仅作内部字段/兼容用途,不参与路由)。
func deriveProvider(formats []string) string {
if len(formats) == 0 {
return store.ChannelProviderCompatible
}
messagesOnly, hasResponses := true, false
for _, f := range formats {
if f != store.FormatMessages {
messagesOnly = false
}
if f == store.FormatResponses {
hasResponses = true
}
}
if messagesOnly {
return store.ChannelProviderAnthropic
}
if hasResponses {
return store.ChannelProviderOpenAI
}
return store.ChannelProviderCompatible
}
// resolveFormats 渠道协议格式:显式给出则校验去重;空则按 provider 推断默认。
func resolveFormats(provider string, formats []string) ([]string, error) {
if len(formats) == 0 {
switch provider {
case store.ChannelProviderAnthropic:
return []string{store.FormatMessages}, nil
case store.ChannelProviderOpenAI:
return []string{store.FormatChat, store.FormatResponses}, nil
default:
return []string{store.FormatChat}, nil
}
}
seen := map[string]bool{}
out := make([]string, 0, len(formats))
for _, f := range formats {
if !validFormats[f] {
return nil, fmt.Errorf("unsupported format %q", f)
}
if !seen[f] {
seen[f] = true
out = append(out, f)
}
}
return out, nil
}
// AdminCreateChannel POST /api/v1/admin/channels
func (h *Handler) AdminCreateChannel(c *gin.Context) {
var req channelBody
if err := c.ShouldBindJSON(&req); err != nil {
resp.Fail(c, http.StatusBadRequest, "invalid input: "+err.Error())
return
}
if req.Provider == "" {
req.Provider = deriveProvider(req.Formats)
}
if !validateProvider(req.Provider) {
resp.Fail(c, http.StatusBadRequest, "provider must be openai, anthropic or compatible")
return
}
if req.APIKey == "" {
resp.Fail(c, http.StatusBadRequest, "api_key required")
return
}
formats, err := resolveFormats(req.Provider, req.Formats)
if err != nil {
resp.Fail(c, http.StatusBadRequest, err.Error())
return
}
baseURL, err := resolveBaseURL(req.Provider, req.BaseURL)
if err != nil {
resp.Fail(c, http.StatusBadRequest, err.Error())
return
}
enc, err := h.a.Enc.Encrypt(req.APIKey)
if err != nil {
resp.Fail(c, http.StatusInternalServerError, "failed to encrypt api key")
return
}
ch := store.Channel{
Name: req.Name, Provider: req.Provider, Formats: formats, BaseURL: baseURL,
BaseURLs: normalizeBaseURLs(req.BaseURLs),
APIKeyEnc: enc, Weight: intOr(req.Weight, 1), Priority: intOr(req.Priority, 0),
TimeoutMS: intOr(req.TimeoutMS, 300000), MaxConcurrency: intOr(req.MaxConcurrency, 16),
HealthStatus: store.ChannelHealthHealthy, Enabled: boolOr(req.Enabled, true),
}
if err := h.a.DB.Create(&ch).Error; err != nil {
resp.Fail(c, http.StatusConflict, "failed to create channel (name may already exist)")
return
}
resp.Created(c, gin.H{"id": ch.ID, "name": ch.Name})
}
// AdminUpdateChannel PUT /api/v1/admin/channels/:id
func (h *Handler) AdminUpdateChannel(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
resp.Fail(c, http.StatusBadRequest, "invalid channel id")
return
}
var body struct {
Name *string `json:"name"`
Provider *string `json:"provider"`
Formats *[]string `json:"formats"`
BaseURL *string `json:"base_url"`
BaseURLs *map[string]string `json:"base_urls"`
APIKey *string `json:"api_key"`
Weight *int `json:"weight"`
Priority *int `json:"priority"`
TimeoutMS *int `json:"timeout_ms"`
MaxConcurrency *int `json:"max_concurrency"`
HealthStatus *string `json:"health_status"`
Enabled *bool `json:"enabled"`
}
if err := c.ShouldBindJSON(&body); err != nil {
resp.Fail(c, http.StatusBadRequest, "invalid input")
return
}
var ch store.Channel
if err := h.a.DB.First(&ch, id).Error; err != nil {
resp.Fail(c, http.StatusNotFound, "channel not found")
return
}
updates := map[string]any{}
if body.Name != nil {
updates["name"] = *body.Name
}
if body.Provider != nil {
if !validateProvider(*body.Provider) {
resp.Fail(c, http.StatusBadRequest, "provider must be openai, anthropic or compatible")
return
}
updates["provider"] = *body.Provider
}
if body.BaseURL != nil {
prov := ch.Provider
if body.Provider != nil {
prov = *body.Provider
}
b, berr := resolveBaseURL(prov, *body.BaseURL)
if berr != nil {
resp.Fail(c, http.StatusBadRequest, berr.Error())
return
}
updates["base_url"] = b
}
if body.BaseURLs != nil {
// base_urls 是 jsonb:手动序列化
raw, _ := json.Marshal(normalizeBaseURLs(*body.BaseURLs))
updates["base_urls"] = string(raw)
}
if body.APIKey != nil && *body.APIKey != "" {
enc, err := h.a.Enc.Encrypt(*body.APIKey)
if err != nil {
resp.Fail(c, http.StatusInternalServerError, "failed to encrypt api key")
return
}
updates["api_key_enc"] = enc
}
if body.Weight != nil {
updates["weight"] = *body.Weight
}
if body.Priority != nil {
updates["priority"] = *body.Priority
}
if body.TimeoutMS != nil {
updates["timeout_ms"] = *body.TimeoutMS
}
if body.MaxConcurrency != nil {
updates["max_concurrency"] = *body.MaxConcurrency
}
if body.HealthStatus != nil {
updates["health_status"] = *body.HealthStatus
}
if body.Enabled != nil {
updates["enabled"] = *body.Enabled
}
if body.Formats != nil {
prov := ch.Provider
if body.Provider != nil {
prov = *body.Provider
}
formats, ferr := resolveFormats(prov, *body.Formats)
if ferr != nil {
resp.Fail(c, http.StatusBadRequest, ferr.Error())
return
}
// formats 是 jsonb:手动序列化为 JSON 字符串(map 更新不走序列化)
raw, _ := json.Marshal(formats)
updates["formats"] = string(raw)
}
if len(updates) > 0 {
if err := h.a.DB.Model(&ch).Updates(updates).Error; err != nil {
resp.Fail(c, http.StatusInternalServerError, "failed to update channel")
return
}
}
resp.OK(c, gin.H{"ok": true})
}
// AdminDeleteChannel DELETE /api/v1/admin/channels/:id
func (h *Handler) AdminDeleteChannel(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
resp.Fail(c, http.StatusBadRequest, "invalid channel id")
return
}
res := h.a.DB.Delete(&store.Channel{}, id)
if res.Error != nil {
resp.Fail(c, http.StatusInternalServerError, "failed to delete channel")
return
}
if res.RowsAffected == 0 {
resp.Fail(c, http.StatusNotFound, "channel not found")
return
}
// 清理模型绑定
h.a.DB.Where("channel_id = ?", id).Delete(&store.ChannelModelBinding{})
resp.OK(c, gin.H{"ok": true})
}
// AdminTestChannel POST /api/v1/admin/channels/:id/test — 请求渠道 /v1/models 测连通性。
func (h *Handler) AdminTestChannel(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
resp.Fail(c, http.StatusBadRequest, "invalid channel id")
return
}
var ch store.Channel
if err := h.a.DB.First(&ch, id).Error; err != nil {
resp.Fail(c, http.StatusNotFound, "channel not found")
return
}
key, err := h.a.Enc.Decrypt(ch.APIKeyEnc)
if err != nil {
resp.Fail(c, http.StatusInternalServerError, "failed to decrypt channel key")
return
}
url := ch.UpstreamURL("", "/models")
client := &http.Client{Timeout: 10 * time.Second}
req, _ := http.NewRequest(http.MethodGet, url, nil)
req.Header.Set("Authorization", "Bearer "+key)
req.Header.Set("Accept", "application/json")
start := time.Now()
resp2, err := client.Do(req)
status := store.ChannelHealthHealthy
msg := "ok"
latency := 0
if err != nil {
status = store.ChannelHealthCooldown
msg = err.Error()
} else {
latency = int(time.Since(start).Milliseconds())
if resp2.StatusCode < 200 || resp2.StatusCode >= 300 {
status = store.ChannelHealthCooldown
b, _ := io.ReadAll(io.LimitReader(resp2.Body, 1024))
msg = fmt.Sprintf("http %d: %s", resp2.StatusCode, strings.TrimSpace(string(b)))
}
resp2.Body.Close()
}
h.a.DB.Model(&store.Channel{}).Where("id = ?", ch.ID).Update("health_status", status)
if status != store.ChannelHealthHealthy {
resp.Fail(c, http.StatusBadGateway, msg)
return
}
resp.OK(c, gin.H{"ok": true, "latency_ms": latency, "message": msg})
}
// AdminRecoverChannel POST /api/v1/admin/channels/:id/recover — 立即恢复渠道:
// 清掉运行期冷却与连续失败计数,并把探测写的 health_status 置回 healthy。
// 不改 enabled;上游若真挂着,下一次探测或连续 3 次请求失败会重新把它踢掉。
func (h *Handler) AdminRecoverChannel(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
resp.Fail(c, http.StatusBadRequest, "invalid channel id")
return
}
var ch store.Channel
if err := h.a.DB.First(&ch, id).Error; err != nil {
resp.Fail(c, http.StatusNotFound, "channel not found")
return
}
if h.gw != nil {
h.gw.RecoverChannel(ch.ID)
}
if err := h.a.DB.Model(&store.Channel{}).Where("id = ?", ch.ID).
Update("health_status", store.ChannelHealthHealthy).Error; err != nil {
resp.Fail(c, http.StatusInternalServerError, "failed to update channel")
return
}
resp.OK(c, gin.H{"id": ch.ID, "name": ch.Name,
"health_status": store.ChannelHealthHealthy, "runtime_fails": 0, "cooldown_seconds": 0})
}
// maskAPIKey 掩码渠道密钥:保留前 7 位与后 4 位,中间固定 ****** 遮蔽。
// 示例:xxxxxxx******Mq4Y;密钥较短时退化为仅保留后 4 位。
func maskAPIKey(key string) string {
if len(key) <= 11 {
return strings.Repeat("*", len(key)-4) + key[len(key)-4:]
}
return key[:7] + "******" + key[len(key)-4:]
}
func intOr(p *int, def int) int {
if p == nil {
return def
}
return *p
}
func boolOr(p *bool, def bool) bool {
if p == nil {
return def
}
return *p
}