Sakurasan
d5dac5a6fc
llm（三协议 Provider）/ session（JSONL 持久化）/ tools（Registry、围栏、 builtin、computer）/ cdp（browser）/ skills / mcp / agent（循环、两级压缩、 编排、审批门控）+ 底座 jsonl/textx/execx/diag/scratch/spill/imagecache。 产品装配代码（config 桥接、schedule 工具、memory/plan/retro/prompt） 留在 AutoGo，宿主接入方式见 MIGRATION.md。
239 lines
9.0 KiB
Go
239 lines
9.0 KiB
Go
package agent
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"git.gopher.ink/mirrors2/AutoZero/llm"
|
||
"git.gopher.ink/mirrors2/AutoZero/tools"
|
||
)
|
||
|
||
// flakyTool 按入参决定成败：input 含 "fail" 则失败（同时带部分输出）。
|
||
func flakyTool() *tools.Tool {
|
||
return &tools.Tool{
|
||
Name: "flaky",
|
||
Description: "test tool",
|
||
Parameters: map[string]any{"type": "object"},
|
||
Run: func(_ context.Context, input string) (tools.Result, error) {
|
||
if strings.Contains(input, "fail") {
|
||
// 关键：失败但保留 stdout —— 正是 Result.Err 存在的意义。
|
||
return tools.Result{Text: "partial-stdout", Err: errors.New("boom")}, nil
|
||
}
|
||
return tools.TextResult("ok"), nil
|
||
},
|
||
}
|
||
}
|
||
|
||
// TestToolFailureStructured：失败回灌必须是结构化的，且不能丢掉已有输出。
|
||
func TestToolFailureStructured(t *testing.T) {
|
||
a, _ := newTestAgent(t, flakyTool())
|
||
got := a.execResult(context.Background(), llm.ToolCall{ID: "1", Name: "flaky", Input: `{"x":"fail"}`}).res
|
||
|
||
if !strings.Contains(got.Text, "partial-stdout") {
|
||
t.Errorf("partial output lost:\n%s", got.Text)
|
||
}
|
||
if !strings.Contains(got.Text, "[TOOL FAILED] flaky") {
|
||
t.Errorf("missing [TOOL FAILED] marker:\n%s", got.Text)
|
||
}
|
||
// 旧行为是带前导换行的裸 "\nerror: ..."；新实现不应再产生前导换行。
|
||
if strings.HasPrefix(got.Text, "\n") {
|
||
t.Errorf("failure text should not start with a newline: %q", got.Text)
|
||
}
|
||
// 单次失败还不该出现 [STUCK]。
|
||
if strings.Contains(got.Text, "[STUCK]") {
|
||
t.Errorf("first failure must not warn about being stuck:\n%s", got.Text)
|
||
}
|
||
}
|
||
|
||
// TestFailStreakWarnAndReset：连续失败到阈值给出 [STUCK]，成功后清零。
|
||
func TestFailStreakWarnAndReset(t *testing.T) {
|
||
a, _ := newTestAgent(t, flakyTool())
|
||
ctx := context.Background()
|
||
call := func(in string) string {
|
||
return a.execResult(ctx, llm.ToolCall{ID: "1", Name: "flaky", Input: in}).res.Text
|
||
}
|
||
|
||
if got := call(`{"x":"fail"}`); strings.Contains(got, "[STUCK]") {
|
||
t.Errorf("1st failure should not warn:\n%s", got)
|
||
}
|
||
if got := call(`{"x":"fail"}`); strings.Contains(got, "[STUCK]") {
|
||
t.Errorf("2nd failure should not warn:\n%s", got)
|
||
}
|
||
|
||
third := call(`{"x":"fail"}`)
|
||
if !strings.Contains(third, "[STUCK]") || !strings.Contains(third, "3 times in a row") {
|
||
t.Errorf("3rd consecutive failure should warn about being stuck:\n%s", third)
|
||
}
|
||
|
||
// 成功一次 → 计数清零 → 再失败不再告警。
|
||
call(`{"x":"ok"}`)
|
||
after := call(`{"x":"fail"}`)
|
||
if strings.Contains(after, "[STUCK]") {
|
||
t.Errorf("streak should reset after a success:\n%s", after)
|
||
}
|
||
}
|
||
|
||
// TestFailStreakIsPerTool：计数按工具名隔离，一个工具失败不影响另一个。
|
||
func TestFailStreakIsPerTool(t *testing.T) {
|
||
a, _ := newTestAgent(t, flakyTool())
|
||
ctx := context.Background()
|
||
for i := 0; i < 3; i++ {
|
||
a.execResult(ctx, llm.ToolCall{ID: "1", Name: "flaky", Input: `{"x":"fail"}`})
|
||
}
|
||
// 另一个工具首次失败不应继承 flaky 的连续失败计数。
|
||
other := &tools.Tool{
|
||
Name: "other",
|
||
Parameters: map[string]any{"type": "object"},
|
||
Run: func(_ context.Context, _ string) (tools.Result, error) {
|
||
return tools.Result{}, errors.New("nope")
|
||
},
|
||
}
|
||
if err := a.Tools.Register(other); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
got := a.execResult(ctx, llm.ToolCall{ID: "2", Name: "other", Input: "{}"}).res.Text
|
||
if strings.Contains(got, "[STUCK]") {
|
||
t.Errorf("streak must be per tool:\n%s", got)
|
||
}
|
||
}
|
||
|
||
// TestBashExitCodeReported：bash 的非零退出必须成为显式失败信号（Result.Err），
|
||
// 且 stdout 仍然保留。
|
||
func TestBashExitCodeReported(t *testing.T) {
|
||
tl, ok := lookupBash(t)
|
||
if !ok {
|
||
t.Skip("bash tool not registered")
|
||
}
|
||
res, err := tl.Run(context.Background(), `{"command":"echo hello-stdout; exit 3"}`)
|
||
if err != nil {
|
||
t.Fatalf("unexpected error return: %v", err)
|
||
}
|
||
if res.Err == nil {
|
||
t.Fatal("bash must report non-zero exit via Result.Err")
|
||
}
|
||
if !strings.Contains(res.Text, "hello-stdout") {
|
||
t.Errorf("stdout lost on failure: %q", res.Text)
|
||
}
|
||
if kind, _ := classifyFailure(failInfo{Tool: "bash", Elapsed: time.Second}, res.Err); kind != "nonzero-exit" {
|
||
t.Errorf("kind = %q, want nonzero-exit", kind)
|
||
}
|
||
// 成功退出不应带 Err。
|
||
ok2, _ := tl.Run(context.Background(), `{"command":"echo fine"}`)
|
||
if ok2.Err != nil {
|
||
t.Errorf("successful command must not set Err: %v", ok2.Err)
|
||
}
|
||
}
|
||
|
||
func lookupBash(t *testing.T) (*tools.Tool, bool) {
|
||
t.Helper()
|
||
reg := tools.NewRegistry()
|
||
for _, tl := range tools.Builtin() {
|
||
_ = reg.Register(tl)
|
||
}
|
||
return reg.Get("bash")
|
||
}
|
||
|
||
// TestClassifyFailure：覆盖每一条分类规则。
|
||
func TestClassifyFailure(t *testing.T) {
|
||
cases := []struct {
|
||
name string
|
||
in failInfo
|
||
err error
|
||
kind string
|
||
}{
|
||
{"timeout", failInfo{Tool: "bash"}, errors.New("signal: killed"), "timeout"},
|
||
{"deadline", failInfo{Tool: "bash"}, errors.New("context deadline exceeded"), "timeout"},
|
||
{"wait-delay", failInfo{Tool: "bash"}, errors.New("exec: wait delay exceeded"), "timeout"},
|
||
// 同为 signal: killed，用户停止必须归 cancelled：
|
||
// 说成超时就等于指着一条没问题的命令说"它太慢了"。
|
||
{"stopped-mid-bash", failInfo{Tool: "bash", Cancelled: true}, errors.New("signal: killed"), "cancelled"},
|
||
{"cancelled", failInfo{Tool: "bash"}, context.Canceled, "cancelled"},
|
||
{"self-kill", failInfo{Tool: "bash", Input: `{"command":"pkill -f bili-coin-dl; cat /tmp/x.json"}`},
|
||
errors.New("signal: terminated"), "self-kill"},
|
||
// 没有 -f 的按名字杀不会自匹配，别乱给提示。
|
||
{"terminated-other", failInfo{Tool: "bash", Input: `{"command":"make"}`},
|
||
errors.New("signal: terminated"), "unknown"},
|
||
{"danger", failInfo{Tool: "bash"}, errors.New("dangerous action blocked (rm-recursive): ..."), "danger-blocked"},
|
||
{"not-found", failInfo{Tool: "bash"}, errors.New("exit status 127"), "not-found"},
|
||
{"not-exec", failInfo{Tool: "bash"}, errors.New("exit status 126"), "not-executable"},
|
||
{"permission", failInfo{Tool: "read"}, errors.New("open /x: permission denied"), "permission"},
|
||
{"missing-path", failInfo{Tool: "read"}, errors.New("open /nope: no such file or directory"), "missing-path"},
|
||
{"unknown-tool", failInfo{Tool: "bash"}, errors.New(`unknown tool "foo"`), "unknown-tool"},
|
||
{"denied", failInfo{Tool: "bash"}, errors.New(`user denied tool "bash"`), "denied"},
|
||
{"skill", failInfo{Tool: "bash"}, errors.New(`tool "bash" is not allowed by the active skill`), "skill-allowlist"},
|
||
{"bad-args", failInfo{Tool: "bash"}, errors.New("invalid tool arguments: bad json"), "bad-args"},
|
||
{"fallback", failInfo{Tool: "bash"}, errors.New("something odd"), "unknown"},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
c.in.Elapsed = 42 * time.Second
|
||
if c.in.Tool == "" {
|
||
c.in.Tool = "bash"
|
||
}
|
||
kind, hint := classifyFailure(c.in, c.err)
|
||
if kind != c.kind {
|
||
t.Errorf("kind = %q, want %q", kind, c.kind)
|
||
}
|
||
if hint == "" {
|
||
t.Error("hint must not be empty")
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestSelfKillHintIsActionable：提示必须给出一条能立刻照做的写法。
|
||
func TestSelfKillHintIsActionable(t *testing.T) {
|
||
_, hint := classifyFailure(failInfo{Tool: "bash",
|
||
Input: `{"command":"pkill -f bili-coin-dl; echo killed"}`}, errors.New("signal: terminated"))
|
||
for _, want := range []string{"[b]ili", "command line", "bash_bg"} {
|
||
if !strings.Contains(hint, want) {
|
||
t.Errorf("提示里缺 %q: %s", want, hint)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestTimeoutHintReportsReality：超时提示只准说真实发生过的时间。
|
||
// 旧文案对所有工具硬写 "bash limit 120s"：fetch 超时也被说成 bash，
|
||
// 而实际挂了 23 分钟的那次仍然报 120s（session 20261008-214717）。
|
||
func TestTimeoutHintReportsReality(t *testing.T) {
|
||
_, hint := classifyFailure(failInfo{Tool: "fetch", Elapsed: 20 * time.Second},
|
||
errors.New("context deadline exceeded"))
|
||
if strings.Contains(hint, "bash") {
|
||
t.Errorf("fetch 的超时提示里出现了 bash: %q", hint)
|
||
}
|
||
if !strings.Contains(hint, "20s") {
|
||
t.Errorf("提示应包含真实耗时 20s: %q", hint)
|
||
}
|
||
|
||
_, hint = classifyFailure(failInfo{Tool: "bash", Elapsed: 23 * time.Minute},
|
||
errors.New("signal: killed"))
|
||
if strings.Contains(hint, "120s") || !strings.Contains(hint, "23m0s") {
|
||
t.Errorf("提示应报真实耗时而不是固定 120s: %q", hint)
|
||
}
|
||
|
||
block, kind := formatToolFailure(failInfo{Tool: "bash", Elapsed: 23 * time.Minute},
|
||
errors.New("signal: killed"), "", 1)
|
||
if !strings.Contains(block, "23m0s") {
|
||
t.Errorf("失败块应带真实耗时: %q", block)
|
||
}
|
||
if kind != "timeout" {
|
||
t.Errorf("kind = %q, want timeout（这个值会落进 llm.Message.Fail）", kind)
|
||
}
|
||
}
|
||
|
||
// TestDangerBlockHasOwnKind：不可逆动作门的失败要有自己的类别
|
||
// （stats/retro 按 kind 归集摩擦点，混进 unknown 就看不出是被门拦的）。
|
||
func TestDangerBlockHasOwnKind(t *testing.T) {
|
||
block, kind := formatToolFailure(failInfo{Tool: "bash"},
|
||
errors.New("dangerous action blocked (rm-recursive): do not work around it"), "", 1)
|
||
if !strings.Contains(block, "(danger-blocked)") {
|
||
t.Errorf("失败块应标 danger-blocked: %q", block)
|
||
}
|
||
if kind != "danger-blocked" {
|
||
t.Errorf("kind = %q", kind)
|
||
}
|
||
}
|