AutoZero/agent/toolfail.go

package agent

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

// failStreakWarn 是连续失败达到该次数后，在失败块里追加 [STUCK] 换思路提示。
const failStreakWarn = 3

// isTimeoutErr 识别"被时间限制杀掉"的错误形态。
//   - signal: killed / context deadline exceeded：ctx 超时（bash 120s、fetch 20s…）
//   - wait delay exceeded：进程已死，但它派生的进程仍占着输出管道，
//     exec 的 WaitDelay 兜底生效（见 internal/execx）。
func isTimeoutErr(msg string) bool {
	return strings.Contains(msg, "context deadline exceeded") ||
		strings.Contains(msg, "signal: killed") ||
		strings.Contains(msg, "wait delay exceeded")
}

// failInfo 是判断「这次失败到底是怎么回事」所需的上下文。光看错误文本不够：
// 同一个 "signal: killed" 可能是超时，也可能是用户按了停止。
type failInfo struct {
	Tool      string
	Input     string // 调用入参原文——识别 pkill 自匹配只能靠它
	Elapsed   time.Duration
	Cancelled bool // 本回合被用户主动停止（不是超时）
}

// classifyFailure 按错误文本判定失败类别，并给出给模型看的针对性提示。
// 全部为英文：与系统提示词、既有 error 文案保持一致。
//
// Elapsed 只为一件事：**提示里不许出现编造的数字**。旧文案对所有工具硬写
// "bash limit 120s"，而事故那一次 bash 实际挂了 23 分钟——模型据此判断
// "再试一次应该没事"，人看到的也是错的诊断。
func classifyFailure(in failInfo, err error) (kind, hint string) {
	if err == nil {
		return "unknown", ""
	}
	msg := strings.ToLower(err.Error())
	switch {
	case in.Cancelled || strings.Contains(msg, "context canceled"):
		// 用户按了停止：这不是"命令失败"，不能让模型当成失败去重试同一件事。
		// 必须排在 timeout 之前——被停止杀掉的进程报的也是 "signal: killed"，
		// 归成超时就等于告诉模型"这条命令太慢了"，而它其实没问题。
		return "cancelled", "The user stopped this turn, so the call was cancelled rather than failed. " +
			"Do not retry it as if something was wrong with the command."
	case strings.Contains(msg, "signal: terminated") && selfMatchedKiller(in.Input):
		return "self-kill", selfKillHint
	case isTimeoutErr(msg):
		return "timeout", fmt.Sprintf("%s timed out after %s. Narrow the scope "+
			"(limit paths, add --max-count, split the command) instead of rerunning it as-is; "+
			"rerunning it unchanged will just time out again.", in.Tool, in.Elapsed.Round(time.Second))
	case strings.Contains(msg, "dangerous action blocked"):
		return "danger-blocked", "The irreversible-action gate stopped this. Say what you were about to do " +
			"and why, then stop; do not attempt the same effect through another tool or a rewritten command."
	case strings.Contains(msg, "exit status 127"):
		return "not-found", "Command not found (exit 127). Check the spelling or use an absolute path."
	case strings.Contains(msg, "exit status 126"):
		return "not-executable", "Command found but not executable (exit 126). Check permissions or invoke it via its interpreter."
	case strings.Contains(msg, "permission denied"):
		return "permission", "Permission denied. Do not retry with sudo; check the path and whether the operation is allowed."
	case strings.Contains(msg, "no such file or directory"):
		return "missing-path", "Path does not exist. Confirm it with ls/read before retrying."
	case strings.Contains(msg, "unknown tool"):
		return "unknown-tool", "That tool does not exist. Check the available tool list before calling again."
	case strings.Contains(msg, "user denied"):
		return "denied", "The user denied this operation. Do not retry the same thing with a different wording."
	case strings.Contains(msg, "not allowed by the active skill"):
		return "skill-allowlist", "The active skill restricts which tools may be used. Stay within its allowed tools."
	case strings.Contains(msg, "invalid") && (strings.Contains(msg, "arguments") || strings.Contains(msg, "input")):
		return "bad-args", "Arguments did not match the tool schema. Re-read the parameter definition and fix the shape."
	}
	if code := exitCode(err); code > 0 {
		return "nonzero-exit", fmt.Sprintf("Command exited with status %d. Read the output above to find the cause; do not rerun the same command unchanged.", code)
	}
	return "unknown", "The call failed. Read the message above, fix the root cause, then retry differently."
}

// exitCode 从 "exit status N" 形态的错误里取出退出码；取不到返回 -1。
func exitCode(err error) int {
	var ee interface{ ExitCode() int }
	if errors.As(err, &ee) {
		return ee.ExitCode()
	}
	return -1
}

// formatToolFailure 拼装回灌给模型的失败块，并返回归一出的失败类别。
// out 是工具已产出的部分输出（可能为 ""），会原样保留在失败块之前，避免丢失
// 有价值的 stdout。kind 除了进文本，还落进会话消息的结构字段（见 llm.Message.Fail），
// 让 stats/retro 不必再解析字符串前缀。
func formatToolFailure(in failInfo, err error, out string, streak int) (string, string) {
	var b strings.Builder
	if strings.TrimSpace(out) != "" {
		b.WriteString(strings.TrimRight(out, "\n"))
		b.WriteString("\n")
	}
	kind, hint := classifyFailure(in, err)
	fmt.Fprintf(&b, "[TOOL FAILED] %s: %s (%s)\n", in.Tool, err.Error(), kind)
	if hint != "" {
		fmt.Fprintf(&b, "[hint] %s\n", hint)
	}
	if streak >= failStreakWarn {
		fmt.Fprintf(&b, "[STUCK] %s has now failed %d times in a row. Stop repeating the same approach: "+
			"switch tool or change the arguments, re-verify your assumptions, or update the plan.\n", in.Tool, streak)
	}
	return strings.TrimRight(b.String(), "\n"), kind
}

// selfKillHint 针对的是 agent 跑 shell 时一个几乎必中的坑（实测两次，都只报
// unknown，模型看不出门道）。
const selfKillHint = "The command killed the shell that was running it: pkill/killall/pgrep with -f match " +
	"full command lines, and this command's own line contains the pattern you searched for — so it terminated " +
	"itself and nothing after it ran. Avoid self-matching: bracket the first letter (`[b]ili-coin-dl` matches " +
	"the process but not your own command line), match an exact name with -x, kill a recorded pid, or start the " +
	"job with bash_bg and stop it by id."

// selfMatchedKiller 认「用 -f 模式杀进程，而模式串就写在自己的命令行里」。
//
// `sh -c "pkill -f myapp; …"` 的完整命令行包含 "pkill -f myapp"，其中就有 "myapp"，
// 于是 pkill 先把执行它的 shell 杀了（实测 15ms 就 signal: terminated），后面的
// cat/校验/收尾一个字节都没执行。只有 -f（全命令行匹配）有这个问题，-x 按名字匹配不会。
func selfMatchedKiller(input string) bool {
	l := strings.ToLower(input)
	if !strings.Contains(l, "-f") {
		return false
	}
	return strings.Contains(l, "pkill") || strings.Contains(l, "killall") || strings.Contains(l, "pgrep")
}

// noteToolFailure 记录一次工具失败，返回该工具当前的连续失败次数。
// 按**工具名**计数：agent 重试时通常会微调参数，按「工具+参数」会漏判，
// 而全局计数又太噪。
func (a *Agent) noteToolFailure(tool string) int {
	a.failMu.Lock()
	defer a.failMu.Unlock()
	if a.failStreak == nil {
		a.failStreak = map[string]int{}
	}
	a.failStreak[tool]++
	return a.failStreak[tool]
}

// noteToolSuccess 在工具成功时清零该工具的连续失败计数（说明它有进展）。
func (a *Agent) noteToolSuccess(tool string) {
	a.failMu.Lock()
	defer a.failMu.Unlock()
	if a.failStreak == nil {
		return
	}
	delete(a.failStreak, tool)
}