feat(providers): explicit prompt cache for DashScope/Qwen (#1127)

* feat(providers): explicit prompt cache for DashScope/Qwen

Extend OpenAI-compat path with Anthropic-style cache_control:ephemeral
inline blocks for Alibaba DashScope endpoints (provider_type=bailian or
URL contains "dashscope"). Verified live: qwen3.6-plus / qwen3.5-plus /
qwen3-coder-plus all return 99.7% cache hit rate on 6K-token prefix,
yielding ~89% token cost reduction on cached prefix (90% Alibaba discount).

- isDashScope(): 3-source detection (URL + providerType + name) handles
  reverse-proxied endpoints; covers both "dashscope" and "bailian"
- buildRequestBody wraps system content with SplitSystemPromptForCache
  (exported from anthropic_request.go) using <!-- GOCLAW_CACHE_BOUNDARY -->
- Tool prefix cache: cache_control on last tool definition, with 4-marker
  budget guard
- Parse cache_creation_input_tokens from prompt_tokens_details into
  Usage.CacheCreationTokens; propagates via existing span metadata writers
- Runtime escape hatch: GOCLAW_DISABLE_DASHSCOPE_CACHE=true
- Live integration smoke test (build tag integration, env-gated)

* chore(providers): polish per PR #1127 review

- Use strings.Repeat instead of custom repeat() helper in smoke test
- Clarify BuildRequestBodyForTest is test-only, not public API

* feat(providers): enable thinking for Qwen 3.7/3.6, observe cache in smoke test

Qwen3.7-plus and qwen3.6-plus support deep thinking but were missing from
dashscopeThinkingModels, so enable_thinking/thinking_budget was silently
skipped. Add both to the whitelist and test.

Add qwen3.7-plus to the cache smoke test as cache-optional: Alibaba's
context-cache doc does not yet list 3.7/3.6 for explicit cache, and the
cache wrap is a safe no-op when unsupported, so a no-cache result is logged
rather than failed to avoid a flaky live assertion.

Claude-Session: https://claude.ai/code/session_01X3jkrc7N3ar8ZGzUJWyNS5

* fix(providers): preserve DashScope detection for proxy routes
This commit is contained in:
Plateau Nguyen authored and GitHub committed 2026-06-19 10:59:02 +07:00
1 parent c70324f032
commit 0e994f959c
18 files changed
+689 -22

No files matched your search

@@ -8,7 +8,7 @@ import (
// the cache boundary marker is split into 2 blocks: stable (cached) + dynamic.
func TestAnthropicSystemBlocksSplit(t *testing.T) {
prompt := "stable content\n" + CacheBoundaryMarker + "\ndynamic content"
blocks := splitSystemPromptForCache(prompt)
blocks := SplitSystemPromptForCache(prompt)
if len(blocks) != 2 {
t.Fatalf("expected 2 blocks, got %d", len(blocks))
}
@@ -30,7 +30,7 @@ func TestAnthropicSystemBlocksSplit(t *testing.T) {
// marker → single block with cache_control.
func TestAnthropicSingleBlockFallback(t *testing.T) {
prompt := "no boundary here"
blocks := splitSystemPromptForCache(prompt)
blocks := SplitSystemPromptForCache(prompt)
if len(blocks) != 1 {
t.Fatalf("expected 1 block, got %d", len(blocks))
}
@@ -46,7 +46,7 @@ func TestAnthropicSingleBlockFallback(t *testing.T) {
// after the boundary produces only 1 block (no empty block appended).
func TestAnthropicEmptyDynamic(t *testing.T) {
prompt := "stable only\n" + CacheBoundaryMarker + "\n"
blocks := splitSystemPromptForCache(prompt)
blocks := SplitSystemPromptForCache(prompt)
if len(blocks) != 1 {
t.Fatalf("expected 1 block for empty dynamic, got %d", len(blocks))
}
@@ -59,7 +59,7 @@ func TestAnthropicEmptyDynamic(t *testing.T) {
// (empty stable section) still produces valid blocks without empty text.
func TestAnthropicEmptyStable(t *testing.T) {
prompt := CacheBoundaryMarker + "\ndynamic only"
blocks := splitSystemPromptForCache(prompt)
blocks := SplitSystemPromptForCache(prompt)
// Stable is empty string after TrimSpace — should still produce a block
// (Anthropic API handles empty text blocks gracefully).
if len(blocks) < 1 {
+4 -3
View File
@@ -10,10 +10,11 @@ import (
// to avoid circular import; agent_test verifies they match).
const CacheBoundaryMarker = "<!-- GOCLAW_CACHE_BOUNDARY -->"
// splitSystemPromptForCache splits a system prompt at the cache boundary marker.
// SplitSystemPromptForCache splits a system prompt at CacheBoundaryMarker.
// Returns 2 blocks if boundary found: stable (with cache_control) + dynamic (without).
// Returns 1 block with cache_control if no boundary (backwards compat).
func splitSystemPromptForCache(content string) []map[string]any {
// Used by both Anthropic and DashScope cache middleware (identical wire format).
func SplitSystemPromptForCache(content string) []map[string]any {
ephemeral := map[string]any{"type": "ephemeral"}
before, after, ok := strings.Cut(content, CacheBoundaryMarker)
if !ok {
@@ -95,7 +96,7 @@ func (p *AnthropicProvider) buildRequestBody(model string, req ChatRequest, stre
for _, msg := range req.Messages {
switch msg.Role {
case "system":
systemBlocks = append(systemBlocks, splitSystemPromptForCache(msg.Content)...)
systemBlocks = append(systemBlocks, SplitSystemPromptForCache(msg.Content)...)
case "user":
if len(msg.Images) > 0 || len(msg.Videos) > 0 {
+8 -5
View File
@@ -12,15 +12,18 @@ const (
)
// dashscopeThinkingModels lists DashScope models that accept the
// enable_thinking / thinking_budget parameters (Qwen3 open-weight and Qwen3.5 series).
// enable_thinking / thinking_budget parameters (Qwen3 open-weight and Qwen3.5+ series).
// Models NOT in this set (e.g. qwen3-plus, qwen3-turbo) will silently
// skip thinking injection to avoid API "model not supported" errors.
var dashscopeThinkingModels = map[string]bool{
// Qwen3.7 / 3.6 series — deep thinking + vision
"qwen3.7-plus": true,
"qwen3.6-plus": true,
// Qwen3.5 series — thinking + vision
"qwen3.5-plus": true,
"qwen3.5-turbo": true,
"qwen3.5-plus": true,
"qwen3.5-turbo": true,
// Qwen3 hosted
"qwen3-max": true,
"qwen3-max": true,
// Qwen3 open-weight (available as hosted inference)
"qwen3-235b-a22b": true,
"qwen3-32b": true,
@@ -43,7 +46,7 @@ func NewDashScopeProvider(name, apiKey, apiBase, defaultModel string) *DashScope
defaultModel = dashscopeDefaultModel
}
return &DashScopeProvider{
OpenAIProvider: NewOpenAIProvider(name, apiKey, apiBase, defaultModel),
OpenAIProvider: NewOpenAIProvider(name, apiKey, apiBase, defaultModel).WithProviderType("dashscope"),
}
}
@@ -0,0 +1,62 @@
package providers
import "os"
// wrapSystemForDashScopeCache transforms a system message string content into
// Anthropic-style content blocks with cache_control:ephemeral markers.
//
// DashScope verified 2026-05-08 to accept and process this wire format on
// coding-intl.dashscope.aliyuncs.com. Result: 90% discount on cached prefix
// tokens, 5min sliding TTL.
//
// Non-system messages and non-string content pass through unchanged
// (idempotent, supports already-blocked input).
func wrapSystemForDashScopeCache(msg map[string]any) map[string]any {
if msg["role"] != "system" {
return msg
}
content, ok := msg["content"].(string)
if !ok {
return msg
}
msg["content"] = SplitSystemPromptForCache(content)
return msg
}
// applyDashScopeToolPrefixCache adds cache_control:ephemeral to the last tool
// definition, caching the entire tool prefix (descriptions, schemas).
//
// alreadyMarked: cache markers already consumed by system message blocks.
// DashScope limits 4 markers/request; skip tool marker if limit reached.
func applyDashScopeToolPrefixCache(tools []map[string]any, alreadyMarked int) []map[string]any {
if len(tools) == 0 || alreadyMarked >= 4 {
return tools
}
last := tools[len(tools)-1]
last["cache_control"] = map[string]any{"type": "ephemeral"}
return tools
}
// countCacheControlMarkers counts cache_control fields in a message's content
// blocks. Used to track marker budget across system + tools.
func countCacheControlMarkers(msg map[string]any) int {
blocks, ok := msg["content"].([]map[string]any)
if !ok {
return 0
}
count := 0
for _, b := range blocks {
if b["cache_control"] != nil {
count++
}
}
return count
}
// dashScopeCacheDisabled returns true when env var GOCLAW_DISABLE_DASHSCOPE_CACHE
// is set to a truthy value. Provides runtime escape hatch without requiring
// code redeploy or config change.
func dashScopeCacheDisabled() bool {
v := os.Getenv("GOCLAW_DISABLE_DASHSCOPE_CACHE")
return v == "true" || v == "1" || v == "yes"
}
@@ -0,0 +1,207 @@
package providers
import (
"encoding/json"
"reflect"
"testing"
)
func TestWrapSystemForDashScopeCache_NoBoundary(t *testing.T) {
msg := map[string]any{
"role": "system",
"content": "You are a helpful assistant.",
}
out := wrapSystemForDashScopeCache(msg)
blocks, ok := out["content"].([]map[string]any)
if !ok {
t.Fatalf("expected []map content, got %T", out["content"])
}
if len(blocks) != 1 {
t.Fatalf("expected 1 block, got %d", len(blocks))
}
if blocks[0]["cache_control"] == nil {
t.Error("block[0] missing cache_control")
}
}
func TestWrapSystemForDashScopeCache_WithBoundary(t *testing.T) {
msg := map[string]any{
"role": "system",
"content": "Stable prefix\n" + CacheBoundaryMarker + "\nDynamic suffix",
}
out := wrapSystemForDashScopeCache(msg)
blocks := out["content"].([]map[string]any)
if len(blocks) != 2 {
t.Fatalf("expected 2 blocks, got %d", len(blocks))
}
if blocks[0]["cache_control"] == nil {
t.Error("stable block missing cache_control")
}
if blocks[1]["cache_control"] != nil {
t.Error("dynamic block should not have cache_control")
}
}
func TestWrapSystemForDashScopeCache_NonSystemUntouched(t *testing.T) {
msg := map[string]any{
"role": "user",
"content": "Hello",
}
out := wrapSystemForDashScopeCache(msg)
if !reflect.DeepEqual(out, msg) {
t.Error("user message should pass through unchanged")
}
}
func TestWrapSystemForDashScopeCache_NonStringContentUntouched(t *testing.T) {
blocks := []map[string]any{{"type": "text", "text": "x", "cache_control": map[string]any{"type": "ephemeral"}}}
msg := map[string]any{"role": "system", "content": blocks}
out := wrapSystemForDashScopeCache(msg)
got, _ := json.Marshal(out["content"])
want, _ := json.Marshal(blocks)
if string(got) != string(want) {
t.Errorf("idempotent fail: got %s want %s", got, want)
}
}
func TestBuildRequestBody_DashScopeEndpoint_WrapsSystem(t *testing.T) {
p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus")
req := ChatRequest{
Messages: []Message{
{Role: "system", Content: "You are an assistant."},
{Role: "user", Content: "Hi"},
},
}
body := p.buildRequestBody("qwen3.6-plus", req, false)
msgs := body["messages"].([]map[string]any)
sysContent := msgs[0]["content"]
if _, ok := sysContent.([]map[string]any); !ok {
t.Fatalf("expected DashScope system content as []block, got %T", sysContent)
}
}
func TestBuildRequestBody_OpenAINative_DoesNotWrap(t *testing.T) {
p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4o")
req := ChatRequest{Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}}
body := p.buildRequestBody("gpt-4o", req, false)
msgs := body["messages"].([]map[string]any)
if _, ok := msgs[0]["content"].(string); !ok {
t.Errorf("OpenAI native should keep string content, got %T", msgs[0]["content"])
}
}
func TestApplyDashScopeToolPrefixCache_AddMarkerOnLast(t *testing.T) {
tools := []map[string]any{
{"type": "function", "function": map[string]any{"name": "tool_a"}},
{"type": "function", "function": map[string]any{"name": "tool_b"}},
{"type": "function", "function": map[string]any{"name": "tool_c"}},
}
out := applyDashScopeToolPrefixCache(tools, 1)
if len(out) != 3 {
t.Fatalf("len changed: got %d", len(out))
}
if out[0]["cache_control"] != nil || out[1]["cache_control"] != nil {
t.Error("non-last tools should not have cache_control")
}
if out[2]["cache_control"] == nil {
t.Error("last tool missing cache_control")
}
}
func TestApplyDashScopeToolPrefixCache_EmptyArray(t *testing.T) {
out := applyDashScopeToolPrefixCache([]map[string]any{}, 0)
if len(out) != 0 {
t.Errorf("expected empty, got %d", len(out))
}
}
func TestApplyDashScopeToolPrefixCache_RespectsMarkerLimit(t *testing.T) {
tools := []map[string]any{{"type": "function", "function": map[string]any{"name": "x"}}}
out := applyDashScopeToolPrefixCache(tools, 4)
if out[0]["cache_control"] != nil {
t.Error("should skip tool marker when limit reached")
}
}
func TestCountCacheControlMarkers(t *testing.T) {
msg := map[string]any{
"role": "system",
"content": []map[string]any{
{"type": "text", "text": "x", "cache_control": map[string]any{"type": "ephemeral"}},
{"type": "text", "text": "y"},
},
}
if got := countCacheControlMarkers(msg); got != 1 {
t.Errorf("got %d, want 1", got)
}
}
func TestBuildRequestBody_DashScopeWithTools_AppliesToolCache(t *testing.T) {
p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus")
req := ChatRequest{
Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}},
Tools: []ToolDefinition{
{Type: "function", Function: &ToolFunctionSchema{Name: "search", Description: "search docs", Parameters: map[string]any{}}},
{Type: "function", Function: &ToolFunctionSchema{Name: "fetch", Description: "fetch url", Parameters: map[string]any{}}},
},
}
body := p.buildRequestBody("qwen3.6-plus", req, false)
tools := body["tools"].([]map[string]any)
if tools[0]["cache_control"] != nil {
t.Error("first tool should not have cache_control")
}
if tools[len(tools)-1]["cache_control"] == nil {
t.Error("last tool should have cache_control")
}
}
func TestBuildRequestBody_OpenAINativeWithTools_NoToolCache(t *testing.T) {
p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4o")
req := ChatRequest{
Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}},
Tools: []ToolDefinition{
{Type: "function", Function: &ToolFunctionSchema{Name: "x", Description: "y", Parameters: map[string]any{}}},
},
}
body := p.buildRequestBody("gpt-4o", req, false)
tools, ok := body["tools"].([]map[string]any)
if ok && len(tools) > 0 && tools[0]["cache_control"] != nil {
t.Error("OpenAI native should not have tool cache_control")
}
}
func TestBuildRequestBody_DashScopeWithEnvDisable_DoesNotWrap(t *testing.T) {
t.Setenv("GOCLAW_DISABLE_DASHSCOPE_CACHE", "true")
p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus")
req := ChatRequest{Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}}
body := p.buildRequestBody("qwen3.6-plus", req, false)
msgs := body["messages"].([]map[string]any)
if _, ok := msgs[0]["content"].(string); !ok {
t.Errorf("env disable should keep string content, got %T", msgs[0]["content"])
}
}
func TestBuildRequestBody_BailianThinkingLevelMapsToDashScopeKeys(t *testing.T) {
p := NewOpenAIProvider("qwen-richard", "key", "https://proxy.internal/v1", "qwen3.6-plus").
WithProviderType("bailian")
req := ChatRequest{
Model: "qwen3.6-plus",
Messages: []Message{
{Role: "system", Content: "You are an assistant."},
{Role: "user", Content: "Hi"},
},
Options: map[string]any{OptThinkingLevel: "medium"},
}
body := p.buildRequestBody("qwen3.6-plus", req, false)
if body[OptEnableThinking] != true {
t.Fatalf("enable_thinking = %v, want true", body[OptEnableThinking])
}
if body[OptThinkingBudget] == nil {
t.Fatal("thinking_budget missing for Bailian Qwen thinking model")
}
if _, has := body[OptReasoningEffort]; has {
t.Fatal("Bailian must not receive OpenAI reasoning_effort")
}
}
+2
View File
@@ -51,6 +51,8 @@ func TestDashScopeModelSupportsThinking(t *testing.T) {
model string
want bool
}{
{"qwen3.7-plus", true},
{"qwen3.6-plus", true},
{"qwen3.5-plus", true},
{"qwen3.5-turbo", true},
{"qwen3-max", true},
@@ -0,0 +1,57 @@
package providers
import (
"encoding/json"
"testing"
)
func TestOpenAIUsage_DashScopeCacheHit_Unmarshal(t *testing.T) {
raw := `{
"prompt_tokens": 2318,
"completion_tokens": 195,
"total_tokens": 2513,
"prompt_tokens_details": {
"text_tokens": 2318,
"cache_creation_input_tokens": 0,
"cached_tokens": 2304
}
}`
var u openAIUsage
if err := json.Unmarshal([]byte(raw), &u); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if u.PromptTokensDetails.CachedTokens != 2304 {
t.Errorf("cached_tokens: got %d want 2304", u.PromptTokensDetails.CachedTokens)
}
if u.PromptTokensDetails.CacheCreationInputTokens != 0 {
t.Errorf("cache_creation: got %d want 0", u.PromptTokensDetails.CacheCreationInputTokens)
}
}
func TestOpenAIUsage_DashScopeCacheCreate_Unmarshal(t *testing.T) {
raw := `{
"prompt_tokens": 2318,
"prompt_tokens_details": {
"cache_creation_input_tokens": 2304,
"cached_tokens": 0
}
}`
var u openAIUsage
if err := json.Unmarshal([]byte(raw), &u); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if u.PromptTokensDetails.CacheCreationInputTokens != 2304 {
t.Errorf("got %d want 2304", u.PromptTokensDetails.CacheCreationInputTokens)
}
}
func TestOpenAIUsage_NoDetails_OK(t *testing.T) {
raw := `{"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}`
var u openAIUsage
if err := json.Unmarshal([]byte(raw), &u); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if u.PromptTokensDetails != nil {
t.Errorf("expected nil PromptTokensDetails, got %+v", u.PromptTokensDetails)
}
}
+1 -1
View File
@@ -112,7 +112,7 @@ func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChun
}
if chunk.Usage.PromptTokensDetails != nil {
result.Usage.CacheReadTokens = chunk.Usage.PromptTokensDetails.CachedTokens
result.Usage.CacheCreationTokens = chunk.Usage.PromptTokensDetails.CacheWriteTokens
result.Usage.CacheCreationTokens = chunk.Usage.PromptTokensDetails.CacheWriteTokens + chunk.Usage.PromptTokensDetails.CacheCreationInputTokens
result.Usage.PromptTokensIncludeCachedSegments = true
}
if chunk.Usage.CompletionTokensDetails != nil && chunk.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
+17 -3
View File
@@ -41,15 +41,29 @@ func isDashScopeAPIBase(apiBase string) bool {
}
// dashScopePassthroughKeys is true when enable_thinking / thinking_budget may be added to the JSON body.
// Uses URL, provider_type, and name so httptest DashScope URLs still work in tests.
// Uses the same DashScope/Bailian route detection as prompt-cache wrapping.
func (p *OpenAIProvider) dashScopePassthroughKeys() bool {
return p.isDashScope()
}
// isDashScope returns true when this provider routes requests to DashScope/Bailian
// (supports cache_control:ephemeral wire format - verified live 2026-05-08).
// Uses 3-source detection (URL + providerType + name) to handle reverse-proxied
// DashScope endpoints. Includes "bailian" because live qwen-richard provider has
// provider_type=bailian.
//
// Used by buildRequestBody to wrap system content with Anthropic-style
// cache_control blocks for prompt caching (90% discount on cached prefix tokens).
func (p *OpenAIProvider) isDashScope() bool {
if isDashScopeAPIBase(p.apiBase) {
return true
}
if strings.Contains(strings.ToLower(strings.TrimSpace(p.providerType)), "dashscope") {
pt := strings.ToLower(strings.TrimSpace(p.providerType))
if strings.Contains(pt, "dashscope") || strings.Contains(pt, "bailian") {
return true
}
if strings.Contains(strings.ToLower(p.name), "dashscope") {
name := strings.ToLower(p.name)
if strings.Contains(name, "dashscope") || strings.Contains(name, "bailian") {
return true
}
return false
@@ -0,0 +1,57 @@
package providers
import "testing"
func TestOpenAIProvider_isDashScope(t *testing.T) {
cases := []struct {
name string
apiBase string
ptype string
pname string
want bool
}{
{"coding-intl URL", "https://coding-intl.dashscope.aliyuncs.com/v1", "openai_compat", "qwen-richard", true},
{"providerType=bailian", "https://custom-proxy.example.com/v1", "bailian", "internal-qwen", true},
{"providerType=dashscope", "https://proxy.example.com/v1", "dashscope", "x", true},
{"name contains dashscope", "https://proxy.com/v1", "openai_compat", "my-dashscope-mirror", true},
{"name contains bailian", "https://proxy.com/v1", "openai_compat", "company-bailian-relay", true},
{"openai native", "https://api.openai.com/v1", "openai", "gpt", false},
{"anthropic", "https://api.anthropic.com", "anthropic", "claude", false},
{"openrouter", "https://openrouter.ai/api/v1", "openai_compat", "openrouter", false},
{"empty", "", "", "", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
p := &OpenAIProvider{apiBase: tc.apiBase, providerType: tc.ptype, name: tc.pname}
if got := p.isDashScope(); got != tc.want {
t.Errorf("isDashScope() = %v, want %v (apiBase=%q ptype=%q name=%q)",
got, tc.want, tc.apiBase, tc.ptype, tc.pname)
}
})
}
}
func TestOpenAIProvider_dashScopePassthroughKeys(t *testing.T) {
cases := []struct {
name string
apiBase string
ptype string
pname string
want bool
}{
{"dashscope URL", "https://coding-intl.dashscope.aliyuncs.com/v1", "openai_compat", "qwen", true},
{"providerType=dashscope", "https://proxy.example.com/v1", "dashscope", "qwen", true},
{"providerType=bailian", "https://proxy.example.com/v1", "bailian", "qwen", true},
{"name contains dashscope", "https://proxy.example.com/v1", "openai_compat", "team-dashscope", true},
{"name contains bailian", "https://proxy.example.com/v1", "openai_compat", "team-bailian", true},
{"openrouter", "https://openrouter.ai/api/v1", "openai_compat", "openrouter", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
p := &OpenAIProvider{apiBase: tc.apiBase, providerType: tc.ptype, name: tc.pname}
if got := p.dashScopePassthroughKeys(); got != tc.want {
t.Errorf("dashScopePassthroughKeys() = %v, want %v", got, tc.want)
}
})
}
}
+1 -1
View File
@@ -134,7 +134,7 @@ func (p *OpenAIProvider) parseResponse(resp *openAIResponse) *ChatResponse {
}
if resp.Usage.PromptTokensDetails != nil {
result.Usage.CacheReadTokens = resp.Usage.PromptTokensDetails.CachedTokens
result.Usage.CacheCreationTokens = resp.Usage.PromptTokensDetails.CacheWriteTokens
result.Usage.CacheCreationTokens = resp.Usage.PromptTokensDetails.CacheWriteTokens + resp.Usage.PromptTokensDetails.CacheCreationInputTokens
result.Usage.PromptTokensIncludeCachedSegments = true
}
if resp.Usage.CompletionTokensDetails != nil && resp.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
+26
View File
@@ -159,6 +159,15 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream
msgs = append(msgs, msg)
}
// Apply DashScope cache_control wrapping (verified live 2026-05-08).
// Uses 3-source detection from p.isDashScope() (URL + providerType + name)
// to handle reverse-proxied endpoints. No-op for non-DashScope endpoints
// or when env disabled. For native OpenAI, role mapping above renames
// "system"→"developer" so wrap is a no-op (role guard).
if p.isDashScope() && !dashScopeCacheDisabled() && len(msgs) > 0 {
msgs[0] = wrapSystemForDashScopeCache(msgs[0])
}
// Safety net: strip trailing assistant message to prevent HTTP 400 from
// proxy providers (LiteLLM, OpenRouter) that don't support assistant prefill.
// This should rarely trigger — the agent loop ensures user message is last.
@@ -185,6 +194,19 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream
}
}
// DashScope tool prefix cache: cache_control on last tool definition
// caches the entire tools array (descriptions + schemas, ~5-10K tokens).
// Combined with system block cache: 2/4 markers used, 99.5% hit rate verified.
if p.isDashScope() && !dashScopeCacheDisabled() {
if t, ok := body["tools"].([]map[string]any); ok && len(t) > 0 {
markersFromSystem := 0
if len(msgs) > 0 {
markersFromSystem = countCacheControlMarkers(msgs[0])
}
body["tools"] = applyDashScopeToolPrefixCache(t, markersFromSystem)
}
}
// Together returns HTTP 400 on some requests when stream_options is present.
if stream && !p.isTogetherEndpoint() {
body["stream_options"] = map[string]any{
@@ -249,6 +271,10 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream
// DashScope-specific passthrough keys — never send to other OpenAI-compat hosts.
if p.dashScopePassthroughKeys() {
if level, ok := req.Options[OptThinkingLevel].(string); ok && level != "" && level != "off" && dashscopeThinkingModels[model] {
body[OptEnableThinking] = true
body[OptThinkingBudget] = dashscopeThinkingBudget(level)
}
if v, ok := req.Options[OptEnableThinking]; ok {
body[OptEnableThinking] = v
}
@@ -0,0 +1,8 @@
package providers
// BuildRequestBodyForTest is a test-only export of buildRequestBody for
// integration smoke tests in the tests/integration package. NOT part of the
// public API - do not call from production code paths.
func (p *OpenAIProvider) BuildRequestBodyForTest(model string, req ChatRequest, stream bool) map[string]any {
return p.buildRequestBody(model, req, stream)
}
+3 -2
View File
@@ -56,8 +56,9 @@ type openAIUsage struct {
}
type openAIPromptDetails struct {
CachedTokens int `json:"cached_tokens"`
CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
CachedTokens int `json:"cached_tokens"`
CacheWriteTokens int `json:"cache_write_tokens,omitempty"` // some OpenAI-compat providers
CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` // DashScope/Qwen
}
type openAICompletionDetails struct {