mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 04:16:06 +00:00
A run records its terminal run.status (completed/failed/cancelled) only when the agent loop returns. If the gateway process is killed mid-run (restart, crash, or SIGKILL after a slow graceful shutdown), the terminal event is never emitted, so the run's last run.status item stays "started" forever — it shows as perpetually running in the timeline and is never counted as failed. Mirror the cron scheduler's startup reset (recomputeStaleJobs flips stale 'running' jobs to 'interrupted'): add RunTimelineStore.RecoverInterruptedRuns, invoked once on gateway startup. It finds runs that have a started run.status but no terminal sibling and appends a terminal failed run.status item, marked interrupted in metadata so it stays distinguishable from a genuine agent failure. Running only at startup means nothing from before is still executing, so there is no false-positive risk — unlike a periodic sweep (see the deliberately-disabled tracing stale-trace loop). - store: RecoverInterruptedRuns on the interface + PG and SQLite impls - cmd/gateway_managed: invoke once after the timeline recorder is built - tests: SQLite store test — orphan recovered, completed run untouched, idempotent on re-run
176 lines
5.5 KiB
Go
176 lines
5.5 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/nextlevelbuilder/goclaw/internal/store"
|
|
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
|
|
)
|
|
|
|
type recordingTimelineStore struct {
|
|
mu sync.Mutex
|
|
items []store.RunTimelineItem
|
|
}
|
|
|
|
func (s *recordingTimelineStore) AppendRunTimelineItem(_ context.Context, item *store.RunTimelineItem) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.items = append(s.items, *item)
|
|
return nil
|
|
}
|
|
|
|
func (s *recordingTimelineStore) ListRunTimelineItems(context.Context, store.RunTimelineListOpts) ([]store.RunTimelineItem, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (s *recordingTimelineStore) RecoverInterruptedRuns(context.Context) (int64, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
func TestRunTimelineItemFromEventScrubsToolArguments(t *testing.T) {
|
|
tenantID := uuid.Must(uuid.NewV7())
|
|
item, ok := runTimelineItemFromEvent(AgentEvent{
|
|
Type: protocol.AgentEventToolCall,
|
|
AgentID: "default",
|
|
RunID: "run-1",
|
|
UserID: "user-1",
|
|
Channel: "web",
|
|
ChatID: "chat-1",
|
|
SessionKey: "session-1",
|
|
TenantID: tenantID,
|
|
Payload: map[string]any{
|
|
"name": "exec_command",
|
|
"id": "call-1",
|
|
"arguments": map[string]any{"cmd": "echo sk-abcdefghijklmnopqrstuvwxyz123456"},
|
|
},
|
|
}, 7)
|
|
if !ok {
|
|
t.Fatal("expected timeline item")
|
|
}
|
|
if item.ItemType != store.RunTimelineItemTypeToolCall {
|
|
t.Fatalf("ItemType = %q", item.ItemType)
|
|
}
|
|
if item.Seq != 7 {
|
|
t.Fatalf("Seq = %d, want 7", item.Seq)
|
|
}
|
|
if item.ToolName != "exec_command" || item.ToolCallID != "call-1" {
|
|
t.Fatalf("tool fields = %q/%q", item.ToolName, item.ToolCallID)
|
|
}
|
|
if strings.Contains(item.Preview, "sk-abcdefghijklmnopqrstuvwxyz123456") {
|
|
t.Fatalf("preview leaked secret: %s", item.Preview)
|
|
}
|
|
if !strings.Contains(item.Preview, "[REDACTED]") {
|
|
t.Fatalf("preview missing redaction: %s", item.Preview)
|
|
}
|
|
if item.AgentID != nil {
|
|
t.Fatalf("AgentID = %v, want nil for agent key", item.AgentID)
|
|
}
|
|
if !strings.Contains(string(item.Metadata), `"agent_key":"default"`) {
|
|
t.Fatalf("metadata missing agent_key: %s", item.Metadata)
|
|
}
|
|
}
|
|
|
|
func TestRunTimelineItemFromEventDropsUnsupportedAndThinking(t *testing.T) {
|
|
tenantID := uuid.Must(uuid.NewV7())
|
|
if _, ok := runTimelineItemFromEvent(AgentEvent{
|
|
Type: protocol.ChatEventThinking,
|
|
RunID: "run-1",
|
|
SessionKey: "session-1",
|
|
TenantID: tenantID,
|
|
}, 1); ok {
|
|
t.Fatal("thinking event should not be archived")
|
|
}
|
|
|
|
item, ok := runTimelineItemFromEvent(AgentEvent{
|
|
Type: protocol.AgentEventRunCompleted,
|
|
RunID: "run-1",
|
|
SessionKey: "session-1",
|
|
TenantID: tenantID,
|
|
Payload: map[string]any{
|
|
"content": "visible <thinking>hidden chain</thinking> done",
|
|
"thinking": "raw hidden chain",
|
|
},
|
|
}, 1)
|
|
if !ok {
|
|
t.Fatal("expected completed item")
|
|
}
|
|
if strings.Contains(item.Preview, "hidden chain") || strings.Contains(item.Preview, "raw hidden") {
|
|
t.Fatalf("preview leaked thinking: %q", item.Preview)
|
|
}
|
|
if item.Preview != "visible done" {
|
|
t.Fatalf("Preview = %q", item.Preview)
|
|
}
|
|
}
|
|
|
|
func TestRunTimelinePreviewStripsDeliveryFileTokens(t *testing.T) {
|
|
tenantID := uuid.Must(uuid.NewV7())
|
|
item, ok := runTimelineItemFromEvent(AgentEvent{
|
|
Type: protocol.AgentEventRunCompleted,
|
|
RunID: "run-1",
|
|
SessionKey: "session-1",
|
|
TenantID: tenantID,
|
|
Payload: map[string]any{
|
|
"content": "See  and /v1/media/b.txt?x=1&ft=stale.456",
|
|
},
|
|
}, 1)
|
|
if !ok {
|
|
t.Fatal("expected completed item")
|
|
}
|
|
if strings.Contains(item.Preview, "ft=") || strings.Contains(item.Preview, "signed.123") || strings.Contains(item.Preview, "stale.456") {
|
|
t.Fatalf("preview leaked delivery token: %q", item.Preview)
|
|
}
|
|
if !strings.Contains(item.Preview, "/v1/files/work/a.png") || !strings.Contains(item.Preview, "/v1/media/b.txt?x=1") {
|
|
t.Fatalf("preview lost clean file URLs: %q", item.Preview)
|
|
}
|
|
}
|
|
|
|
func TestRunTimelineRecorderOnlyTracksSupportedActiveRuns(t *testing.T) {
|
|
tenantID := uuid.Must(uuid.NewV7())
|
|
recorder := NewRunTimelineRecorder(&recordingTimelineStore{})
|
|
base := AgentEvent{
|
|
RunID: "run-1",
|
|
SessionKey: "session-1",
|
|
TenantID: tenantID,
|
|
}
|
|
|
|
recorder.Record(AgentEvent{Type: protocol.ChatEventThinking, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
|
|
if got := recorderTrackedRuns(recorder); got != 0 {
|
|
t.Fatalf("tracked runs after unsupported event = %d, want 0", got)
|
|
}
|
|
|
|
recorder.Record(AgentEvent{Type: protocol.AgentEventToolCall, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
|
|
if got := recorderTrackedRuns(recorder); got != 1 {
|
|
t.Fatalf("tracked runs after supported event = %d, want 1", got)
|
|
}
|
|
if seq := recorderSeq(recorder, base.RunID); seq != 1 {
|
|
t.Fatalf("seq after first supported event = %d, want 1", seq)
|
|
}
|
|
|
|
recorder.Record(AgentEvent{Type: protocol.ChatEventThinking, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
|
|
if seq := recorderSeq(recorder, base.RunID); seq != 1 {
|
|
t.Fatalf("seq after unsupported event = %d, want 1", seq)
|
|
}
|
|
|
|
recorder.Record(AgentEvent{Type: protocol.AgentEventRunCompleted, RunID: base.RunID, SessionKey: base.SessionKey, TenantID: base.TenantID})
|
|
if got := recorderTrackedRuns(recorder); got != 0 {
|
|
t.Fatalf("tracked runs after terminal event = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func recorderTrackedRuns(r *RunTimelineRecorder) int {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return len(r.nextSeq)
|
|
}
|
|
|
|
func recorderSeq(r *RunTimelineRecorder, runID string) int {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return r.nextSeq[runID]
|
|
}
|