package agent import ( "context" "errors" "strings" "testing" "time" "git.gopher.ink/mirrors2/AutoZero/llm" "git.gopher.ink/mirrors2/AutoZero/tools" ) // flakyTool 按入参决定成败:input 含 "fail" 则失败(同时带部分输出)。 func flakyTool() *tools.Tool { return &tools.Tool{ Name: "flaky", Description: "test tool", Parameters: map[string]any{"type": "object"}, Run: func(_ context.Context, input string) (tools.Result, error) { if strings.Contains(input, "fail") { // 关键:失败但保留 stdout —— 正是 Result.Err 存在的意义。 return tools.Result{Text: "partial-stdout", Err: errors.New("boom")}, nil } return tools.TextResult("ok"), nil }, } } // TestToolFailureStructured:失败回灌必须是结构化的,且不能丢掉已有输出。 func TestToolFailureStructured(t *testing.T) { a, _ := newTestAgent(t, flakyTool()) got := a.execResult(context.Background(), llm.ToolCall{ID: "1", Name: "flaky", Input: `{"x":"fail"}`}).res if !strings.Contains(got.Text, "partial-stdout") { t.Errorf("partial output lost:\n%s", got.Text) } if !strings.Contains(got.Text, "[TOOL FAILED] flaky") { t.Errorf("missing [TOOL FAILED] marker:\n%s", got.Text) } // 旧行为是带前导换行的裸 "\nerror: ...";新实现不应再产生前导换行。 if strings.HasPrefix(got.Text, "\n") { t.Errorf("failure text should not start with a newline: %q", got.Text) } // 单次失败还不该出现 [STUCK]。 if strings.Contains(got.Text, "[STUCK]") { t.Errorf("first failure must not warn about being stuck:\n%s", got.Text) } } // TestFailStreakWarnAndReset:连续失败到阈值给出 [STUCK],成功后清零。 func TestFailStreakWarnAndReset(t *testing.T) { a, _ := newTestAgent(t, flakyTool()) ctx := context.Background() call := func(in string) string { return a.execResult(ctx, llm.ToolCall{ID: "1", Name: "flaky", Input: in}).res.Text } if got := call(`{"x":"fail"}`); strings.Contains(got, "[STUCK]") { t.Errorf("1st failure should not warn:\n%s", got) } if got := call(`{"x":"fail"}`); strings.Contains(got, "[STUCK]") { t.Errorf("2nd failure should not warn:\n%s", got) } third := call(`{"x":"fail"}`) if !strings.Contains(third, "[STUCK]") || !strings.Contains(third, "3 times in a row") { t.Errorf("3rd consecutive failure should warn about being stuck:\n%s", third) } // 成功一次 → 计数清零 → 再失败不再告警。 call(`{"x":"ok"}`) after := call(`{"x":"fail"}`) if strings.Contains(after, "[STUCK]") { t.Errorf("streak should reset after a success:\n%s", after) } } // TestFailStreakIsPerTool:计数按工具名隔离,一个工具失败不影响另一个。 func TestFailStreakIsPerTool(t *testing.T) { a, _ := newTestAgent(t, flakyTool()) ctx := context.Background() for i := 0; i < 3; i++ { a.execResult(ctx, llm.ToolCall{ID: "1", Name: "flaky", Input: `{"x":"fail"}`}) } // 另一个工具首次失败不应继承 flaky 的连续失败计数。 other := &tools.Tool{ Name: "other", Parameters: map[string]any{"type": "object"}, Run: func(_ context.Context, _ string) (tools.Result, error) { return tools.Result{}, errors.New("nope") }, } if err := a.Tools.Register(other); err != nil { t.Fatal(err) } got := a.execResult(ctx, llm.ToolCall{ID: "2", Name: "other", Input: "{}"}).res.Text if strings.Contains(got, "[STUCK]") { t.Errorf("streak must be per tool:\n%s", got) } } // TestBashExitCodeReported:bash 的非零退出必须成为显式失败信号(Result.Err), // 且 stdout 仍然保留。 func TestBashExitCodeReported(t *testing.T) { tl, ok := lookupBash(t) if !ok { t.Skip("bash tool not registered") } res, err := tl.Run(context.Background(), `{"command":"echo hello-stdout; exit 3"}`) if err != nil { t.Fatalf("unexpected error return: %v", err) } if res.Err == nil { t.Fatal("bash must report non-zero exit via Result.Err") } if !strings.Contains(res.Text, "hello-stdout") { t.Errorf("stdout lost on failure: %q", res.Text) } if kind, _ := classifyFailure(failInfo{Tool: "bash", Elapsed: time.Second}, res.Err); kind != "nonzero-exit" { t.Errorf("kind = %q, want nonzero-exit", kind) } // 成功退出不应带 Err。 ok2, _ := tl.Run(context.Background(), `{"command":"echo fine"}`) if ok2.Err != nil { t.Errorf("successful command must not set Err: %v", ok2.Err) } } func lookupBash(t *testing.T) (*tools.Tool, bool) { t.Helper() reg := tools.NewRegistry() for _, tl := range tools.Builtin() { _ = reg.Register(tl) } return reg.Get("bash") } // TestClassifyFailure:覆盖每一条分类规则。 func TestClassifyFailure(t *testing.T) { cases := []struct { name string in failInfo err error kind string }{ {"timeout", failInfo{Tool: "bash"}, errors.New("signal: killed"), "timeout"}, {"deadline", failInfo{Tool: "bash"}, errors.New("context deadline exceeded"), "timeout"}, {"wait-delay", failInfo{Tool: "bash"}, errors.New("exec: wait delay exceeded"), "timeout"}, // 同为 signal: killed,用户停止必须归 cancelled: // 说成超时就等于指着一条没问题的命令说"它太慢了"。 {"stopped-mid-bash", failInfo{Tool: "bash", Cancelled: true}, errors.New("signal: killed"), "cancelled"}, {"cancelled", failInfo{Tool: "bash"}, context.Canceled, "cancelled"}, {"self-kill", failInfo{Tool: "bash", Input: `{"command":"pkill -f bili-coin-dl; cat /tmp/x.json"}`}, errors.New("signal: terminated"), "self-kill"}, // 没有 -f 的按名字杀不会自匹配,别乱给提示。 {"terminated-other", failInfo{Tool: "bash", Input: `{"command":"make"}`}, errors.New("signal: terminated"), "unknown"}, {"danger", failInfo{Tool: "bash"}, errors.New("dangerous action blocked (rm-recursive): ..."), "danger-blocked"}, {"not-found", failInfo{Tool: "bash"}, errors.New("exit status 127"), "not-found"}, {"not-exec", failInfo{Tool: "bash"}, errors.New("exit status 126"), "not-executable"}, {"permission", failInfo{Tool: "read"}, errors.New("open /x: permission denied"), "permission"}, {"missing-path", failInfo{Tool: "read"}, errors.New("open /nope: no such file or directory"), "missing-path"}, {"unknown-tool", failInfo{Tool: "bash"}, errors.New(`unknown tool "foo"`), "unknown-tool"}, {"denied", failInfo{Tool: "bash"}, errors.New(`user denied tool "bash"`), "denied"}, {"skill", failInfo{Tool: "bash"}, errors.New(`tool "bash" is not allowed by the active skill`), "skill-allowlist"}, {"bad-args", failInfo{Tool: "bash"}, errors.New("invalid tool arguments: bad json"), "bad-args"}, {"fallback", failInfo{Tool: "bash"}, errors.New("something odd"), "unknown"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { c.in.Elapsed = 42 * time.Second if c.in.Tool == "" { c.in.Tool = "bash" } kind, hint := classifyFailure(c.in, c.err) if kind != c.kind { t.Errorf("kind = %q, want %q", kind, c.kind) } if hint == "" { t.Error("hint must not be empty") } }) } } // TestSelfKillHintIsActionable:提示必须给出一条能立刻照做的写法。 func TestSelfKillHintIsActionable(t *testing.T) { _, hint := classifyFailure(failInfo{Tool: "bash", Input: `{"command":"pkill -f bili-coin-dl; echo killed"}`}, errors.New("signal: terminated")) for _, want := range []string{"[b]ili", "command line", "bash_bg"} { if !strings.Contains(hint, want) { t.Errorf("提示里缺 %q: %s", want, hint) } } } // TestTimeoutHintReportsReality:超时提示只准说真实发生过的时间。 // 旧文案对所有工具硬写 "bash limit 120s":fetch 超时也被说成 bash, // 而实际挂了 23 分钟的那次仍然报 120s(session 20261008-214717)。 func TestTimeoutHintReportsReality(t *testing.T) { _, hint := classifyFailure(failInfo{Tool: "fetch", Elapsed: 20 * time.Second}, errors.New("context deadline exceeded")) if strings.Contains(hint, "bash") { t.Errorf("fetch 的超时提示里出现了 bash: %q", hint) } if !strings.Contains(hint, "20s") { t.Errorf("提示应包含真实耗时 20s: %q", hint) } _, hint = classifyFailure(failInfo{Tool: "bash", Elapsed: 23 * time.Minute}, errors.New("signal: killed")) if strings.Contains(hint, "120s") || !strings.Contains(hint, "23m0s") { t.Errorf("提示应报真实耗时而不是固定 120s: %q", hint) } block, kind := formatToolFailure(failInfo{Tool: "bash", Elapsed: 23 * time.Minute}, errors.New("signal: killed"), "", 1) if !strings.Contains(block, "23m0s") { t.Errorf("失败块应带真实耗时: %q", block) } if kind != "timeout" { t.Errorf("kind = %q, want timeout(这个值会落进 llm.Message.Fail)", kind) } } // TestDangerBlockHasOwnKind:不可逆动作门的失败要有自己的类别 // (stats/retro 按 kind 归集摩擦点,混进 unknown 就看不出是被门拦的)。 func TestDangerBlockHasOwnKind(t *testing.T) { block, kind := formatToolFailure(failInfo{Tool: "bash"}, errors.New("dangerous action blocked (rm-recursive): do not work around it"), "", 1) if !strings.Contains(block, "(danger-blocked)") { t.Errorf("失败块应标 danger-blocked: %q", block) } if kind != "danger-blocked" { t.Errorf("kind = %q", kind) } }