mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-16 16:15:40 +00:00
feat(webhooks): stream provider responses for server-side runs to enable prompt caching (#1273)
Server-side webhook agent runs (sync, async, and admin test) used Stream:false, so OpenAI-compatible routers that only cache streaming requests never populated or served their prompt cache. Webhook runs paid full input-token price on every turn even with a stable session and an identical multi-turn prefix, while WS chat (Stream:true) got cache hits from the 3rd turn. Add gateway.webhook_stream (default true, env GOCLAW_WEBHOOK_STREAM) and apply it to all three run sites. ChatStream returns the fully assembled response, so the payload returned to the caller is unchanged. Set to false to restore non-streaming.
This commit is contained in:
1 parent
68ad5953d7
commit
e2ec3370dd
11 files changed
+108
-3
No files matched your search
@@ -0,0 +1,18 @@
|
||||
package webhooks
|
||||
|
||||
// DefaultStream is the fallback for the webhook internal-streaming flag when
|
||||
// config is unset. Streaming is on by default so the upstream provider can
|
||||
// populate/serve its prompt cache for server-side webhook agent runs (sync,
|
||||
// async, and test). Non-streaming requests are not cached by some
|
||||
// OpenAI-compatible routers, so webhook runs would otherwise always pay full
|
||||
// input-token price even with a stable session.
|
||||
const DefaultStream = true
|
||||
|
||||
// ResolveStream converts the optional gateway.webhook_stream config into a bool:
|
||||
// nil → DefaultStream (true); otherwise the explicit value.
|
||||
func ResolveStream(v *bool) bool {
|
||||
if v == nil {
|
||||
return DefaultStream
|
||||
}
|
||||
return *v
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package webhooks
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestResolveStream(t *testing.T) {
|
||||
tr := true
|
||||
fa := false
|
||||
cases := []struct {
|
||||
name string
|
||||
in *bool
|
||||
want bool
|
||||
}{
|
||||
{"nil uses default (true)", nil, true},
|
||||
{"explicit true", &tr, true},
|
||||
{"explicit false", &fa, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := ResolveStream(c.in); got != c.want {
|
||||
t.Fatalf("ResolveStream(%v) = %v, want %v", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -117,6 +117,11 @@ type WorkerConfig struct {
|
||||
|
||||
// AsyncAgentTimeout is the deadline for each async agent run. 0 → DefaultAgentTimeout (600s).
|
||||
AsyncAgentTimeout time.Duration
|
||||
|
||||
// Stream controls whether the async agent run streams provider responses so the
|
||||
// upstream can populate/serve its prompt cache. Resolve via webhooks.ResolveStream
|
||||
// (default true) before constructing the config.
|
||||
Stream bool
|
||||
}
|
||||
|
||||
// WebhookWorker is the background callback delivery service. It is started once per
|
||||
@@ -758,7 +763,7 @@ func (w *WebhookWorker) invokeAgent(
|
||||
ChatID: call.WebhookID.String(),
|
||||
RunID: runID,
|
||||
UserID: req.UserID,
|
||||
Stream: false,
|
||||
Stream: w.cfg.Stream,
|
||||
ModelOverride: req.Model,
|
||||
ExtraSystemPrompt: extraSystem,
|
||||
TraceName: "webhook.async",
|
||||
|
||||
Reference in new issue
Block a user