feat: agent heartbeat system — periodic proactive check-ins (#245)

Phase 1 (Core):
- Migration 000022: agent_heartbeats, heartbeat_run_logs, agent_config_permissions tables
- HeartbeatStore + ConfigPermissionStore interfaces with PG implementations
- HeartbeatTicker: background poll → active hours filter → queue-aware skip → run → smart suppression → deliver/log
- Heartbeat tool: status/get/set/toggle/set_checklist/get_checklist/test/logs actions
- Permission check with wildcard scope matching + TTL cache (60s)
- RPC methods: heartbeat.get/set/toggle/test/logs/checklist.get/checklist.set
- HEARTBEAT.md routed via context file interceptor (read/write for both open + predefined agents)
- Session keys: agent:{id}:heartbeat or agent:{id}💓{ts} (isolated)
- PromptMinimal for heartbeat sessions (like cron/subagent)
- Event broadcasting + cache invalidation via bus (heartbeat + config_perms)
- Gateway wiring: ticker init, event wiring, graceful shutdown

Phase 2 (Integration):
- wakeMode: CronPayload.WakeHeartbeat triggers heartbeat after cron job completes
- Queue-aware: Scheduler.HasActiveSessionsForAgent() skips busy agents
- Stagger: deterministic FNV offset spreads heartbeats across interval
- lightContext: RunRequest.LightContext skips context files, only injects checklist
- System prompt distinguishes cron (user-scheduled tasks) vs heartbeat (autonomous monitoring)
This commit is contained in:
viettranx committed 2026-03-18 13:11:44 +07:00
1 parent da59377387
commit 08a2d95c0c
25 files changed
+2088 -16

No files matched your search

+33 -2
View File
@@ -24,6 +24,7 @@ import (
zalopersonal "github.com/nextlevelbuilder/goclaw/internal/channels/zalo/personal"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/gateway"
"github.com/nextlevelbuilder/goclaw/internal/heartbeat"
"github.com/nextlevelbuilder/goclaw/internal/gateway/methods"
httpapi "github.com/nextlevelbuilder/goclaw/internal/http"
mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
@@ -172,6 +173,12 @@ func runGateway() {
toolsReg.Register(tools.NewCronTool(pgStores.Cron))
slog.Info("cron tool registered")
// Heartbeat tool (agent-facing)
heartbeatTool := tools.NewHeartbeatTool(pgStores.Heartbeats, pgStores.ConfigPermissions)
heartbeatTool.SetAgentStore(pgStores.Agents)
toolsReg.Register(heartbeatTool)
slog.Info("heartbeat tool registered")
// Session tools (list, status, history, send)
toolsReg.Register(tools.NewSessionsListTool())
toolsReg.Register(tools.NewSessionStatusTool())
@@ -398,7 +405,7 @@ func runGateway() {
// Register all RPC methods
server.SetLogTee(logTee)
pairingMethods := registerAllMethods(server, agentRouter, pgStores.Sessions, pgStores.Cron, pgStores.Pairing, cfg, cfgPath, workspace, dataDir, msgBus, execApprovalMgr, pgStores.Agents, pgStores.Skills, pgStores.ConfigSecrets, pgStores.Teams, contextFileInterceptor, logTee)
pairingMethods, heartbeatMethods := registerAllMethods(server, agentRouter, pgStores.Sessions, pgStores.Cron, pgStores.Pairing, cfg, cfgPath, workspace, dataDir, msgBus, execApprovalMgr, pgStores.Agents, pgStores.Skills, pgStores.ConfigSecrets, pgStores.Teams, contextFileInterceptor, logTee, pgStores.Heartbeats)
// Wire pairing event broadcasts to all WS clients.
pairingMethods.SetBroadcaster(server.BroadcastEvent)
@@ -700,6 +707,29 @@ func runGateway() {
slog.Warn("cron service failed to start", "error", err)
}
// Start heartbeat ticker (routes through scheduler's cron lane)
heartbeatTicker := heartbeat.NewTicker(heartbeat.TickerConfig{
Store: pgStores.Heartbeats,
Agents: pgStores.Agents,
MsgBus: msgBus,
Sched: sched,
RunAgent: makeHeartbeatRunFn(sched),
})
heartbeatTicker.SetOnEvent(func(event store.HeartbeatEvent) {
server.BroadcastEvent(*protocol.NewEvent(protocol.EventHeartbeat, event))
})
heartbeatTicker.Start()
// Wire heartbeat wake function to tool + RPC + cron wakeMode
heartbeatTool.SetWakeFn(heartbeatTicker.Wake)
heartbeatMethods.SetWakeFn(heartbeatTicker.Wake)
heartbeatMethods.SetAgentStore(pgStores.Agents)
cronHeartbeatWakeFn = func(agentID string) {
if id, err := uuid.Parse(agentID); err == nil {
heartbeatTicker.Wake(id)
}
}
// Adaptive throttle: reduce per-session concurrency when nearing the summary threshold.
// This prevents concurrent runs from racing with summarization.
// Uses calibrated token estimation (actual prompt tokens from last LLM call)
@@ -842,9 +872,10 @@ func runGateway() {
// Broadcast shutdown event
server.BroadcastEvent(*protocol.NewEvent(protocol.EventShutdown, nil))
// Stop channels, cron, and task ticker
// Stop channels, cron, heartbeat, and task ticker
channelMgr.StopAll(context.Background())
pgStores.Cron.Stop()
heartbeatTicker.Stop()
if taskTicker != nil {
taskTicker.Stop()
}
+15
View File
@@ -0,0 +1,15 @@
package cmd
import (
"context"
"github.com/nextlevelbuilder/goclaw/internal/agent"
"github.com/nextlevelbuilder/goclaw/internal/scheduler"
)
// makeHeartbeatRunFn creates a function that routes a heartbeat run through the scheduler's cron lane.
func makeHeartbeatRunFn(sched *scheduler.Scheduler) func(ctx context.Context, req agent.RunRequest) <-chan scheduler.RunOutcome {
return func(ctx context.Context, req agent.RunRequest) <-chan scheduler.RunOutcome {
return sched.Schedule(ctx, scheduler.LaneCron, req)
}
}
+28
View File
@@ -358,6 +358,34 @@ func wireExtras(
})
}
// Heartbeat cache: invalidate due cache on config changes
if hi, ok := stores.Heartbeats.(store.CacheInvalidatable); ok {
msgBus.Subscribe(bus.TopicCacheHeartbeat, func(event bus.Event) {
if event.Name != protocol.EventCacheInvalidate {
return
}
payload, ok := event.Payload.(bus.CacheInvalidatePayload)
if !ok || payload.Kind != bus.CacheKindHeartbeat {
return
}
hi.InvalidateCache()
})
}
// Config permissions cache: invalidate on grant/revoke changes
if pi, ok := stores.ConfigPermissions.(store.CacheInvalidatable); ok {
msgBus.Subscribe(bus.TopicCacheConfigPerms, func(event bus.Event) {
if event.Name != protocol.EventCacheInvalidate {
return
}
payload, ok := event.Payload.(bus.CacheInvalidatePayload)
if !ok || payload.Kind != bus.CacheKindConfigPerms {
return
}
pi.InvalidateCache()
})
}
// Custom tools cache: reload global tools on create/update/delete
if dynamicLoader != nil {
msgBus.Subscribe(bus.TopicCacheCustomTools, func(event bus.Event) {
+7 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/tools"
)
func registerAllMethods(server *gateway.Server, agents *agent.Router, sessStore store.SessionStore, cronStore store.CronStore, pairingStore store.PairingStore, cfg *config.Config, cfgPath, workspace, dataDir string, msgBus *bus.MessageBus, execApprovalMgr *tools.ExecApprovalManager, agentStore store.AgentStore, skillStore store.SkillStore, configSecretsStore store.ConfigSecretsStore, teamStore store.TeamStore, contextFileInterceptor *tools.ContextFileInterceptor, logTee *gateway.LogTee) *methods.PairingMethods {
func registerAllMethods(server *gateway.Server, agents *agent.Router, sessStore store.SessionStore, cronStore store.CronStore, pairingStore store.PairingStore, cfg *config.Config, cfgPath, workspace, dataDir string, msgBus *bus.MessageBus, execApprovalMgr *tools.ExecApprovalManager, agentStore store.AgentStore, skillStore store.SkillStore, configSecretsStore store.ConfigSecretsStore, teamStore store.TeamStore, contextFileInterceptor *tools.ContextFileInterceptor, logTee *gateway.LogTee, heartbeatStore store.HeartbeatStore) (*methods.PairingMethods, *methods.HeartbeatMethods) {
router := server.Router()
// Phase 1: Core methods
@@ -27,6 +27,10 @@ func registerAllMethods(server *gateway.Server, agents *agent.Router, sessStore
// Phase 2: Cron (store created externally, shared with gateway)
methods.NewCronMethods(cronStore, msgBus).Register(router)
// Phase 2: Heartbeat
heartbeatMethods := methods.NewHeartbeatMethods(heartbeatStore, msgBus)
heartbeatMethods.Register(router)
// Phase 2: Pairing (store created externally, shared with channel manager).
// OnApprove callback is set later by the caller after channel manager is created.
pairingMethods := methods.NewPairingMethods(pairingStore, msgBus, server.RateLimiter())
@@ -51,8 +55,8 @@ func registerAllMethods(server *gateway.Server, agents *agent.Router, sessStore
slog.Info("registered all RPC methods",
"phase1", []string{"chat", "agents", "sessions", "config"},
"phase2", []string{"skills", "cron", "pairing", "usage", "exec_approval", "send"},
"phase2", []string{"skills", "cron", "heartbeat", "pairing", "usage", "exec_approval", "send"},
)
return pairingMethods
return pairingMethods, heartbeatMethods
}