Files
goclaw/internal/agent/run_timeline_recorder_test.go
Zezae Oh 505b7a9d99 fix(agent): recover interrupted runs to failed on gateway startup (#1272)
A run records its terminal run.status (completed/failed/cancelled) only
when the agent loop returns. If the gateway process is killed mid-run
(restart, crash, or SIGKILL after a slow graceful shutdown), the terminal
event is never emitted, so the run's last run.status item stays "started"
forever — it shows as perpetually running in the timeline and is never
counted as failed.

Mirror the cron scheduler's startup reset (recomputeStaleJobs flips stale
'running' jobs to 'interrupted'): add RunTimelineStore.RecoverInterruptedRuns,
invoked once on gateway startup. It finds runs that have a started run.status
but no terminal sibling and appends a terminal failed run.status item, marked
interrupted in metadata so it stays distinguishable from a genuine agent
failure. Running only at startup means nothing from before is still
executing, so there is no false-positive risk — unlike a periodic sweep
(see the deliberately-disabled tracing stale-trace loop).

- store: RecoverInterruptedRuns on the interface + PG and SQLite impls
- cmd/gateway_managed: invoke once after the timeline recorder is built
- tests: SQLite store test — orphan recovered, completed run untouched,
  idempotent on re-run
2026-06-24 14:03:00 +07:00

176 lines
5.5 KiB
Go

package agent
import (
"context"
"strings"
"sync"
"testing"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
type recordingTimelineStore struct {
mu sync.Mutex
items []store.RunTimelineItem
}
func (s *recordingTimelineStore) AppendRunTimelineItem(_ context.Context, item *store.RunTimelineItem) error {
s.mu.Lock()
defer s.mu.Unlock()
s.items = append(s.items, *item)
return nil
}
func (s *recordingTimelineStore) ListRunTimelineItems(context.Context, store.RunTimelineListOpts) ([]store.RunTimelineItem, error) {
return nil, nil
}
func (s *recordingTimelineStore) RecoverInterruptedRuns(context.Context) (int64, error) {
return 0, nil
}
func TestRunTimelineItemFromEventScrubsToolArguments(t *testing.T) {
tenantID := uuid.Must(uuid.NewV7())
item, ok := runTimelineItemFromEvent(AgentEvent{
Type: protocol.AgentEventToolCall,
AgentID: "default",
RunID: "run-1",
UserID: "user-1",
Channel: "web",
ChatID: "chat-1",
SessionKey: "session-1",
TenantID: tenantID,
Payload: map[string]any{
"name": "exec_command",
"id": "call-1",
"arguments": map[string]any{"cmd": "echo sk-abcdefghijklmnopqrstuvwxyz123456"},
},
}, 7)
if !ok {
t.Fatal("expected timeline item")
}
if item.ItemType != store.RunTimelineItemTypeToolCall {
t.Fatalf("ItemType = %q", item.ItemType)
}
if item.Seq != 7 {
t.Fatalf("Seq = %d, want 7", item.Seq)
}
if item.ToolName != "exec_command" || item.ToolCallID != "call-1" {
t.Fatalf("tool fields = %q/%q", item.ToolName, item.ToolCallID)
}
if strings.Contains(item.Preview, "sk-abcdefghijklmnopqrstuvwxyz123456") {
t.Fatalf("preview leaked secret: %s", item.Preview)
}
if !strings.Contains(item.Preview, "[REDACTED]") {
t.Fatalf("preview missing redaction: %s", item.Preview)
}
if item.AgentID != nil {
t.Fatalf("AgentID = %v, want nil for agent key", item.AgentID)
}
if !strings.Contains(string(item.Metadata), `"agent_key":"default"`) {
t.Fatalf("metadata missing agent_key: %s", item.Metadata)
}
}
func TestRunTimelineItemFromEventDropsUnsupportedAndThinking(t *testing.T) {
tenantID := uuid.Must(uuid.NewV7())
if _, ok := runTimelineItemFromEvent(AgentEvent{
Type: protocol.ChatEventThinking,
RunID: "run-1",
SessionKey: "session-1",
TenantID: tenantID,
}, 1); ok {
t.Fatal("thinking event should not be archived")
}
item, ok := runTimelineItemFromEvent(AgentEvent{
Type: protocol.AgentEventRunCompleted,
RunID: "run-1",
SessionKey: "session-1",
TenantID: tenantID,
Payload: map[string]any{
"content": "visible <thinking>hidden chain</thinking> done",
"thinking": "raw hidden chain",
},
}, 1)
if !ok {
t.Fatal("expected completed item")
}
if strings.Contains(item.Preview, "hidden chain") || strings.Contains(item.Preview, "raw hidden") {
t.Fatalf("preview leaked thinking: %q", item.Preview)
}
if item.Preview != "visible done" {
t.Fatalf("Preview = %q", item.Preview)
}
}
func TestRunTimelinePreviewStripsDeliveryFileTokens(t *testing.T) {
tenantID := uuid.Must(uuid.NewV7())
item, ok := runTimelineItemFromEvent(AgentEvent{
Type: protocol.AgentEventRunCompleted,
RunID: "run-1",
SessionKey: "session-1",
TenantID: tenantID,
Payload: map[string]any{
"content": "See ![a](/v1/files/work/a.png?ft=signed.123) and /v1/media/b.txt?x=1&ft=stale.456",
},
}, 1)
if !ok {
t.Fatal("expected completed item")
}
if strings.Contains(item.Preview, "ft=") || strings.Contains(item.Preview, "signed.123") || strings.Contains(item.Preview, "stale.456") {
t.Fatalf("preview leaked delivery token: %q", item.Preview)
}
if !strings.Contains(item.Preview, "/v1/files/work/a.png") || !strings.Contains(item.Preview, "/v1/media/b.txt?x=1") {
t.Fatalf("preview lost clean file URLs: %q", item.Preview)
}
}
func TestRunTimelineRecorderOnlyTracksSupportedActiveRuns(t *testing.T) {
tenantID := uuid.Must(uuid.NewV7())
recorder := NewRunTimelineRecorder(&recordingTimelineStore{})
base := AgentEvent{
RunID: "run-1",
SessionKey: "session-1",
TenantID: tenantID,
}
recorder.Record(AgentEvent{Type: protocol.ChatEventThinking, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
if got := recorderTrackedRuns(recorder); got != 0 {
t.Fatalf("tracked runs after unsupported event = %d, want 0", got)
}
recorder.Record(AgentEvent{Type: protocol.AgentEventToolCall, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
if got := recorderTrackedRuns(recorder); got != 1 {
t.Fatalf("tracked runs after supported event = %d, want 1", got)
}
if seq := recorderSeq(recorder, base.RunID); seq != 1 {
t.Fatalf("seq after first supported event = %d, want 1", seq)
}
recorder.Record(AgentEvent{Type: protocol.ChatEventThinking, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
if seq := recorderSeq(recorder, base.RunID); seq != 1 {
t.Fatalf("seq after unsupported event = %d, want 1", seq)
}
recorder.Record(AgentEvent{Type: protocol.AgentEventRunCompleted, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
if got := recorderTrackedRuns(recorder); got != 0 {
t.Fatalf("tracked runs after terminal event = %d, want 0", got)
}
}
func recorderTrackedRuns(r *RunTimelineRecorder) int {
r.mu.Lock()
defer r.mu.Unlock()
return len(r.nextSeq)
}
func recorderSeq(r *RunTimelineRecorder, runID string) int {
r.mu.Lock()
defer r.mu.Unlock()
return r.nextSeq[runID]
}