Files
viettranx 1ac08155b0 feat(trace): reliable stop/abort with ctx-aware streams and 2-phase router
Makes the Stop button on the traces page actually stop running traces.
Seven-phase implementation across provider HTTP, agent router, trace
persistence, WS events, tool exec, i18n, and integration tests.

- Provider HTTP+SSE ctx-aware: close socket on cancel via CtxBody wrapper
- Router 2-phase abort: CAS state machine, 3s grace, force-mark fallback
- Trace retry: 3 inline retries + 10-max retry queue, stale recovery 10min
- trace.status WS event: real-time UI updates (invalidates query on receive)
- Tool exec: process-group kill (SIGTERM→3s→SIGKILL), Rod page ctx watch
- i18n: 6 abort toast variants in en/vi/zh
- Integration: 9 scenarios, -race clean

Fixes tenant-ctx loss in forceMarkTraceAborted and retry worker broadcast
(caught by code-reviewer: C1/C2). Stale threshold intentionally 10min
because start_time-based; last_span_at migration is a follow-up.
2026-04-14 18:28:31 +07:00

135 lines
3.9 KiB
Go

//go:build integration
package integration
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
var errSimulatedFailure = errors.New("simulated database failure")
// newStallingSSEServer returns an httptest server that sends SSE headers
// then blocks indefinitely, simulating an unresponsive LLM endpoint.
func newStallingSSEServer(t *testing.T) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(http.StatusOK)
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
// Block until client closes connection.
<-r.Context().Done()
}))
}
// newStallingHTTPServer returns an httptest server that accepts connections
// but never sends any bytes, simulating a server that hangs before headers.
// Useful for testing ResponseHeaderTimeout.
func newStallingHTTPServer(t *testing.T) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Block forever without sending headers
<-r.Context().Done()
}))
}
// flakyTracingStore wraps a real TracingStore and fails the first N UpdateTrace
// calls, then succeeds thereafter. Atomic counter allows concurrent test calls.
type flakyTracingStore struct {
store.TracingStore
failsRemaining int32
}
// NewFlakyTracingStore creates a wrapper that fails N times then succeeds.
func newFlakyTracingStore(base store.TracingStore, failCount int) *flakyTracingStore {
return &flakyTracingStore{
TracingStore: base,
failsRemaining: int32(failCount),
}
}
// UpdateTrace fails the first failCount calls, then delegates to the wrapped store.
func (s *flakyTracingStore) UpdateTrace(ctx context.Context, id uuid.UUID, updates map[string]any) error {
fails := atomic.AddInt32(&s.failsRemaining, -1)
if fails >= 0 {
return errSimulatedFailure
}
return s.TracingStore.UpdateTrace(ctx, id, updates)
}
// captureBusEvent subscribes to a message bus and forwards events matching
// the given name into a channel. Automatically unsubscribes on test cleanup.
func captureBusEvent(t *testing.T, msgBus *bus.MessageBus, eventName string) <-chan bus.Event {
t.Helper()
captured := make(chan bus.Event, 10)
subID := "test-sub-" + uuid.New().String()[:8]
msgBus.Subscribe(subID, func(e bus.Event) {
if e.Name == eventName {
select {
case captured <- e:
default:
// channel full, drop (shouldn't happen in tests)
}
}
})
t.Cleanup(func() {
msgBus.Unsubscribe(subID)
close(captured)
})
return captured
}
// mockTraceCollector implements agent.TraceCollector for testing.
type mockTraceCollector struct {
finishCalls []struct {
TraceID uuid.UUID
TenantID uuid.UUID // tenant extracted from ctx at call time
Status string
ErrMsg string
OutputPreview string
}
}
// FinishTrace records the call for later assertion.
func (m *mockTraceCollector) FinishTrace(ctx context.Context, traceID uuid.UUID, status, errMsg, outputPreview string) {
m.finishCalls = append(m.finishCalls, struct {
TraceID uuid.UUID
TenantID uuid.UUID
Status string
ErrMsg string
OutputPreview string
}{traceID, store.TenantIDFromContext(ctx), status, errMsg, outputPreview})
}
// LastFinishTrace returns the most recent FinishTrace call, or nil if none.
func (m *mockTraceCollector) LastFinishTrace() *struct {
TraceID uuid.UUID
TenantID uuid.UUID
Status string
ErrMsg string
OutputPreview string
} {
if len(m.finishCalls) == 0 {
return nil
}
return &m.finishCalls[len(m.finishCalls)-1]
}
// FinishCallCount returns the number of FinishTrace calls.
func (m *mockTraceCollector) FinishCallCount() int {
return len(m.finishCalls)
}