fix: add panic recovery to tool, cron, and summarization goroutines (#420)

* fix: add panic recovery to tool, cron, and summarization goroutines

A panicking tool, cron job, or auto-summarization goroutine crashes
the entire server process — disconnecting all clients and aborting
all running agents. Add defer/recover at three levels:

- safeExecute() in tools/registry.go: catches panics from any tool's
  Execute method and returns an error result to the LLM
- Agent loop goroutine (loop.go): defense-in-depth for non-tool code
  in the parallel tool execution goroutine
- Cron execution goroutine (service_execution.go): prevents a single
  panicking cron job from taking down the server
- Summarization goroutine (loop_history.go): prevents background
  auto-summarization panics from crashing the process

All recovery points log the panic value and stack trace at ERROR level
for debugging. Tool execution panics return actionable error messages
to the LLM so the agent can recover gracefully.

* refactor: extract panic recovery into safego.Recover helper

DRY the duplicated recover+stack+log boilerplate across 4 goroutine
sites into a shared safego.Recover(onPanic, attrs...) function.
Stack buffer bumped from 4096 to 8192 bytes.

---------

Co-authored-by: viettranx <viettranx@gmail.com>
This commit is contained in:
Duc Nguyenandviettranx authored and GitHub committed 2026-03-24 21:09:16 +07:00
1 parent ebc82d3265
commit af3fe2fe08
7 files changed
+224 -2

No files matched your search

+13 -1
View File
@@ -20,6 +20,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/safego"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
@@ -39,10 +40,13 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
l.emit(event)
}
// Inject agent UUID into context for tool routing
// Inject agent UUID + key into context for tool routing
if l.agentUUID != uuid.Nil {
ctx = store.WithAgentID(ctx, l.agentUUID)
}
if l.id != "" {
ctx = store.WithAgentKey(ctx, l.id)
}
// Inject tenant into context for tool-level tenant scoping (spawn, MCP, etc.)
if l.tenantID != uuid.Nil {
ctx = store.WithTenantID(ctx, l.tenantID)
@@ -1140,6 +1144,14 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
wg.Add(1)
go func(idx int, tc providers.ToolCall) {
defer wg.Done()
defer safego.Recover(func(v any) {
resultCh <- indexedResult{
idx: idx,
tc: tc,
registryName: tc.Name,
result: tools.ErrorResult(fmt.Sprintf("tool %q panicked: %v", tc.Name, v)),
}
}, "agent", l.id, "tool", tc.Name)
argsJSON, _ := json.Marshal(tc.Arguments)
slog.Info("tool call", "agent", l.id, "tool", tc.Name, "args_len", len(argsJSON), "parallel", true)
spanStart := time.Now().UTC()