feat(webhooks): stream provider responses for server-side runs to enable prompt caching (#1273)

Server-side webhook agent runs (sync, async, and admin test) used Stream:false,
so OpenAI-compatible routers that only cache streaming requests never populated
or served their prompt cache. Webhook runs paid full input-token price on every
turn even with a stable session and an identical multi-turn prefix, while WS chat
(Stream:true) got cache hits from the 3rd turn.

Add gateway.webhook_stream (default true, env GOCLAW_WEBHOOK_STREAM) and apply it
to all three run sites. ChatStream returns the fully assembled response, so the
payload returned to the caller is unchanged. Set to false to restore non-streaming.
This commit is contained in:
thotam authored and GitHub committed 2026-06-24 13:42:20 +07:00
1 parent 68ad5953d7
commit e2ec3370dd
11 files changed
+108 -3

No files matched your search

+9 -2
View File
@@ -104,6 +104,11 @@ type WebhookLLMHandler struct {
// syncTimeout overrides the webhookLLMTimeout default (600s); set from
// gateway.webhook_sync_timeout_sec config (and in tests). 0 → use default.
syncTimeout time.Duration
// stream controls whether sync + test webhook agent runs stream provider
// responses so the upstream can populate/serve its prompt cache. The response
// returned to the caller is unchanged (assembled JSON). Set from
// gateway.webhook_stream config via webhooks.ResolveStream (default true).
stream bool
}
// NewWebhookLLMHandler constructs a WebhookLLMHandler.
@@ -115,6 +120,7 @@ func NewWebhookLLMHandler(
limiter *webhookLimiter,
lane *scheduler.Lane,
syncTimeout time.Duration,
stream bool,
) *WebhookLLMHandler {
if lane == nil {
lane = scheduler.NewLane(webhookLaneName, webhookLaneDefaultConcurrency)
@@ -126,6 +132,7 @@ func NewWebhookLLMHandler(
limiter: limiter,
lane: lane,
syncTimeout: syncTimeout,
stream: stream,
}
}
@@ -301,7 +308,7 @@ func (h *WebhookLLMHandler) handleSync(
ChatID: webhook.ID.String(),
RunID: runID,
UserID: req.UserID,
Stream: false,
Stream: h.stream,
ModelOverride: req.Model,
ExtraSystemPrompt: extraSystemPrompt,
HistoryLimit: 0,
@@ -562,7 +569,7 @@ func (h *WebhookLLMHandler) RunTest(ctx context.Context, wh *store.WebhookData,
Channel: "webhook",
ChatID: wh.ID.String(),
RunID: runID,
Stream: false,
Stream: h.stream,
ModelOverride: model,
TraceName: "webhook.llm.test",
TraceTags: []string{"webhook", "test"},