Files
Zezae Oh 505b7a9d99 fix(agent): recover interrupted runs to failed on gateway startup (#1272)
A run records its terminal run.status (completed/failed/cancelled) only
when the agent loop returns. If the gateway process is killed mid-run
(restart, crash, or SIGKILL after a slow graceful shutdown), the terminal
event is never emitted, so the run's last run.status item stays "started"
forever — it shows as perpetually running in the timeline and is never
counted as failed.

Mirror the cron scheduler's startup reset (recomputeStaleJobs flips stale
'running' jobs to 'interrupted'): add RunTimelineStore.RecoverInterruptedRuns,
invoked once on gateway startup. It finds runs that have a started run.status
but no terminal sibling and appends a terminal failed run.status item, marked
interrupted in metadata so it stays distinguishable from a genuine agent
failure. Running only at startup means nothing from before is still
executing, so there is no false-positive risk — unlike a periodic sweep
(see the deliberately-disabled tracing stale-trace loop).

- store: RecoverInterruptedRuns on the interface + PG and SQLite impls
- cmd/gateway_managed: invoke once after the timeline recorder is built
- tests: SQLite store test — orphan recovered, completed run untouched,
  idempotent on re-run
2026-06-24 14:03:00 +07:00

120 lines
3.9 KiB
Go

package methods
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/gateway"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
type stubRunTimelineStore struct {
opts []store.RunTimelineListOpts
items []store.RunTimelineItem
err error
}
func (s *stubRunTimelineStore) AppendRunTimelineItem(context.Context, *store.RunTimelineItem) error {
return nil
}
func (s *stubRunTimelineStore) ListRunTimelineItems(_ context.Context, opts store.RunTimelineListOpts) ([]store.RunTimelineItem, error) {
s.opts = append(s.opts, opts)
if s.err != nil {
return nil, s.err
}
return s.items, nil
}
func (s *stubRunTimelineStore) RecoverInterruptedRuns(context.Context) (int64, error) {
return 0, nil
}
func TestRunTimelineGetScopesViewerByUser(t *testing.T) {
timeline := &stubRunTimelineStore{
items: []store.RunTimelineItem{
{RunID: "run-1", UserID: "caller", Seq: 1, Preview: "visible"},
{RunID: "run-1", UserID: "other", Seq: 2, Preview: "hidden"},
},
}
m := NewRunTimelineMethods(timeline, &config.Config{})
tenantID := uuid.Must(uuid.NewV7())
client, responses := gateway.NewCapturingTestClient(permissions.RoleViewer, tenantID, "caller", 1)
ctx := store.WithTenantID(context.Background(), tenantID)
m.handleGet(ctx, client, sessionReqFrame(t, protocol.MethodRunTimelineGet, map[string]any{"runId": "run-1"}))
resp := readTimelineResponse(t, responses)
if resp.Error != nil {
t.Fatalf("unexpected error: %+v", resp.Error)
}
data, ok := resp.Payload.(map[string]any)
if !ok {
t.Fatalf("payload type = %T", resp.Payload)
}
rawItems, ok := data["items"].([]any)
if !ok {
t.Fatalf("items type = %T", data["items"])
}
if len(rawItems) != 1 {
t.Fatalf("len(items) = %d, want 1", len(rawItems))
}
if timeline.opts[0].RunID != "run-1" {
t.Fatalf("RunID = %q", timeline.opts[0].RunID)
}
}
func TestRunTimelineGetRejectsNegativeOffset(t *testing.T) {
timeline := &stubRunTimelineStore{}
m := NewRunTimelineMethods(timeline, &config.Config{})
tenantID := uuid.Must(uuid.NewV7())
client, responses := gateway.NewCapturingTestClient(permissions.RoleViewer, tenantID, "caller", 1)
ctx := store.WithTenantID(context.Background(), tenantID)
m.handleGet(ctx, client, sessionReqFrame(t, protocol.MethodRunTimelineGet, map[string]any{
"runId": "run-1",
"offset": -1,
}))
resp := readTimelineResponse(t, responses)
if resp.Error == nil || resp.Error.Code != protocol.ErrInvalidRequest {
t.Fatalf("error = %+v, want INVALID_REQUEST", resp.Error)
}
if len(timeline.opts) != 0 {
t.Fatalf("store called with opts: %+v", timeline.opts)
}
}
func TestRunTimelineGetHidesStoreErrorDetail(t *testing.T) {
timeline := &stubRunTimelineStore{err: errors.New("pq: syntax error at or near \"OFFSET -1\"")}
m := NewRunTimelineMethods(timeline, &config.Config{})
tenantID := uuid.Must(uuid.NewV7())
client, responses := gateway.NewCapturingTestClient(permissions.RoleViewer, tenantID, "caller", 1)
ctx := store.WithTenantID(context.Background(), tenantID)
m.handleGet(ctx, client, sessionReqFrame(t, protocol.MethodRunTimelineGet, map[string]any{"runId": "run-1"}))
resp := readTimelineResponse(t, responses)
if resp.Error == nil || resp.Error.Code != protocol.ErrInternal {
t.Fatalf("error = %+v, want INTERNAL_ERROR", resp.Error)
}
if strings.Contains(resp.Error.Message, "OFFSET -1") || strings.Contains(resp.Error.Message, "pq:") {
t.Fatalf("error leaked store detail: %q", resp.Error.Message)
}
}
func readTimelineResponse(t *testing.T, ch <-chan []byte) protocol.ResponseFrame {
t.Helper()
raw := <-ch
var resp protocol.ResponseFrame
if err := json.Unmarshal(raw, &resp); err != nil {
t.Fatalf("decode response: %v", err)
}
return resp
}