mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-19 12:16:28 +00:00
feat(providers): explicit prompt cache for DashScope/Qwen (#1127)
* feat(providers): explicit prompt cache for DashScope/Qwen Extend OpenAI-compat path with Anthropic-style cache_control:ephemeral inline blocks for Alibaba DashScope endpoints (provider_type=bailian or URL contains "dashscope"). Verified live: qwen3.6-plus / qwen3.5-plus / qwen3-coder-plus all return 99.7% cache hit rate on 6K-token prefix, yielding ~89% token cost reduction on cached prefix (90% Alibaba discount). - isDashScope(): 3-source detection (URL + providerType + name) handles reverse-proxied endpoints; covers both "dashscope" and "bailian" - buildRequestBody wraps system content with SplitSystemPromptForCache (exported from anthropic_request.go) using <!-- GOCLAW_CACHE_BOUNDARY --> - Tool prefix cache: cache_control on last tool definition, with 4-marker budget guard - Parse cache_creation_input_tokens from prompt_tokens_details into Usage.CacheCreationTokens; propagates via existing span metadata writers - Runtime escape hatch: GOCLAW_DISABLE_DASHSCOPE_CACHE=true - Live integration smoke test (build tag integration, env-gated) * chore(providers): polish per PR #1127 review - Use strings.Repeat instead of custom repeat() helper in smoke test - Clarify BuildRequestBodyForTest is test-only, not public API * feat(providers): enable thinking for Qwen 3.7/3.6, observe cache in smoke test Qwen3.7-plus and qwen3.6-plus support deep thinking but were missing from dashscopeThinkingModels, so enable_thinking/thinking_budget was silently skipped. Add both to the whitelist and test. Add qwen3.7-plus to the cache smoke test as cache-optional: Alibaba's context-cache doc does not yet list 3.7/3.6 for explicit cache, and the cache wrap is a safe no-op when unsupported, so a no-cache result is logged rather than failed to avoid a flaky live assertion. Claude-Session: https://claude.ai/code/session_01X3jkrc7N3ar8ZGzUJWyNS5 * fix(providers): preserve DashScope detection for proxy routes
This commit is contained in:
1 parent
c70324f032
commit
0e994f959c
18 files changed
+689
-22
No files matched your search
@@ -8,7 +8,7 @@ import (
|
||||
// the cache boundary marker is split into 2 blocks: stable (cached) + dynamic.
|
||||
func TestAnthropicSystemBlocksSplit(t *testing.T) {
|
||||
prompt := "stable content\n" + CacheBoundaryMarker + "\ndynamic content"
|
||||
blocks := splitSystemPromptForCache(prompt)
|
||||
blocks := SplitSystemPromptForCache(prompt)
|
||||
if len(blocks) != 2 {
|
||||
t.Fatalf("expected 2 blocks, got %d", len(blocks))
|
||||
}
|
||||
@@ -30,7 +30,7 @@ func TestAnthropicSystemBlocksSplit(t *testing.T) {
|
||||
// marker → single block with cache_control.
|
||||
func TestAnthropicSingleBlockFallback(t *testing.T) {
|
||||
prompt := "no boundary here"
|
||||
blocks := splitSystemPromptForCache(prompt)
|
||||
blocks := SplitSystemPromptForCache(prompt)
|
||||
if len(blocks) != 1 {
|
||||
t.Fatalf("expected 1 block, got %d", len(blocks))
|
||||
}
|
||||
@@ -46,7 +46,7 @@ func TestAnthropicSingleBlockFallback(t *testing.T) {
|
||||
// after the boundary produces only 1 block (no empty block appended).
|
||||
func TestAnthropicEmptyDynamic(t *testing.T) {
|
||||
prompt := "stable only\n" + CacheBoundaryMarker + "\n"
|
||||
blocks := splitSystemPromptForCache(prompt)
|
||||
blocks := SplitSystemPromptForCache(prompt)
|
||||
if len(blocks) != 1 {
|
||||
t.Fatalf("expected 1 block for empty dynamic, got %d", len(blocks))
|
||||
}
|
||||
@@ -59,7 +59,7 @@ func TestAnthropicEmptyDynamic(t *testing.T) {
|
||||
// (empty stable section) still produces valid blocks without empty text.
|
||||
func TestAnthropicEmptyStable(t *testing.T) {
|
||||
prompt := CacheBoundaryMarker + "\ndynamic only"
|
||||
blocks := splitSystemPromptForCache(prompt)
|
||||
blocks := SplitSystemPromptForCache(prompt)
|
||||
// Stable is empty string after TrimSpace — should still produce a block
|
||||
// (Anthropic API handles empty text blocks gracefully).
|
||||
if len(blocks) < 1 {
|
||||
|
||||
@@ -10,10 +10,11 @@ import (
|
||||
// to avoid circular import; agent_test verifies they match).
|
||||
const CacheBoundaryMarker = "<!-- GOCLAW_CACHE_BOUNDARY -->"
|
||||
|
||||
// splitSystemPromptForCache splits a system prompt at the cache boundary marker.
|
||||
// SplitSystemPromptForCache splits a system prompt at CacheBoundaryMarker.
|
||||
// Returns 2 blocks if boundary found: stable (with cache_control) + dynamic (without).
|
||||
// Returns 1 block with cache_control if no boundary (backwards compat).
|
||||
func splitSystemPromptForCache(content string) []map[string]any {
|
||||
// Used by both Anthropic and DashScope cache middleware (identical wire format).
|
||||
func SplitSystemPromptForCache(content string) []map[string]any {
|
||||
ephemeral := map[string]any{"type": "ephemeral"}
|
||||
before, after, ok := strings.Cut(content, CacheBoundaryMarker)
|
||||
if !ok {
|
||||
@@ -95,7 +96,7 @@ func (p *AnthropicProvider) buildRequestBody(model string, req ChatRequest, stre
|
||||
for _, msg := range req.Messages {
|
||||
switch msg.Role {
|
||||
case "system":
|
||||
systemBlocks = append(systemBlocks, splitSystemPromptForCache(msg.Content)...)
|
||||
systemBlocks = append(systemBlocks, SplitSystemPromptForCache(msg.Content)...)
|
||||
|
||||
case "user":
|
||||
if len(msg.Images) > 0 || len(msg.Videos) > 0 {
|
||||
|
||||
@@ -12,15 +12,18 @@ const (
|
||||
)
|
||||
|
||||
// dashscopeThinkingModels lists DashScope models that accept the
|
||||
// enable_thinking / thinking_budget parameters (Qwen3 open-weight and Qwen3.5 series).
|
||||
// enable_thinking / thinking_budget parameters (Qwen3 open-weight and Qwen3.5+ series).
|
||||
// Models NOT in this set (e.g. qwen3-plus, qwen3-turbo) will silently
|
||||
// skip thinking injection to avoid API "model not supported" errors.
|
||||
var dashscopeThinkingModels = map[string]bool{
|
||||
// Qwen3.7 / 3.6 series — deep thinking + vision
|
||||
"qwen3.7-plus": true,
|
||||
"qwen3.6-plus": true,
|
||||
// Qwen3.5 series — thinking + vision
|
||||
"qwen3.5-plus": true,
|
||||
"qwen3.5-turbo": true,
|
||||
"qwen3.5-plus": true,
|
||||
"qwen3.5-turbo": true,
|
||||
// Qwen3 hosted
|
||||
"qwen3-max": true,
|
||||
"qwen3-max": true,
|
||||
// Qwen3 open-weight (available as hosted inference)
|
||||
"qwen3-235b-a22b": true,
|
||||
"qwen3-32b": true,
|
||||
@@ -43,7 +46,7 @@ func NewDashScopeProvider(name, apiKey, apiBase, defaultModel string) *DashScope
|
||||
defaultModel = dashscopeDefaultModel
|
||||
}
|
||||
return &DashScopeProvider{
|
||||
OpenAIProvider: NewOpenAIProvider(name, apiKey, apiBase, defaultModel),
|
||||
OpenAIProvider: NewOpenAIProvider(name, apiKey, apiBase, defaultModel).WithProviderType("dashscope"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package providers
|
||||
|
||||
import "os"
|
||||
|
||||
// wrapSystemForDashScopeCache transforms a system message string content into
|
||||
// Anthropic-style content blocks with cache_control:ephemeral markers.
|
||||
//
|
||||
// DashScope verified 2026-05-08 to accept and process this wire format on
|
||||
// coding-intl.dashscope.aliyuncs.com. Result: 90% discount on cached prefix
|
||||
// tokens, 5min sliding TTL.
|
||||
//
|
||||
// Non-system messages and non-string content pass through unchanged
|
||||
// (idempotent, supports already-blocked input).
|
||||
func wrapSystemForDashScopeCache(msg map[string]any) map[string]any {
|
||||
if msg["role"] != "system" {
|
||||
return msg
|
||||
}
|
||||
content, ok := msg["content"].(string)
|
||||
if !ok {
|
||||
return msg
|
||||
}
|
||||
msg["content"] = SplitSystemPromptForCache(content)
|
||||
return msg
|
||||
}
|
||||
|
||||
// applyDashScopeToolPrefixCache adds cache_control:ephemeral to the last tool
|
||||
// definition, caching the entire tool prefix (descriptions, schemas).
|
||||
//
|
||||
// alreadyMarked: cache markers already consumed by system message blocks.
|
||||
// DashScope limits 4 markers/request; skip tool marker if limit reached.
|
||||
func applyDashScopeToolPrefixCache(tools []map[string]any, alreadyMarked int) []map[string]any {
|
||||
if len(tools) == 0 || alreadyMarked >= 4 {
|
||||
return tools
|
||||
}
|
||||
last := tools[len(tools)-1]
|
||||
last["cache_control"] = map[string]any{"type": "ephemeral"}
|
||||
return tools
|
||||
}
|
||||
|
||||
// countCacheControlMarkers counts cache_control fields in a message's content
|
||||
// blocks. Used to track marker budget across system + tools.
|
||||
func countCacheControlMarkers(msg map[string]any) int {
|
||||
blocks, ok := msg["content"].([]map[string]any)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, b := range blocks {
|
||||
if b["cache_control"] != nil {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// dashScopeCacheDisabled returns true when env var GOCLAW_DISABLE_DASHSCOPE_CACHE
|
||||
// is set to a truthy value. Provides runtime escape hatch without requiring
|
||||
// code redeploy or config change.
|
||||
func dashScopeCacheDisabled() bool {
|
||||
v := os.Getenv("GOCLAW_DISABLE_DASHSCOPE_CACHE")
|
||||
return v == "true" || v == "1" || v == "yes"
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWrapSystemForDashScopeCache_NoBoundary(t *testing.T) {
|
||||
msg := map[string]any{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant.",
|
||||
}
|
||||
out := wrapSystemForDashScopeCache(msg)
|
||||
blocks, ok := out["content"].([]map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected []map content, got %T", out["content"])
|
||||
}
|
||||
if len(blocks) != 1 {
|
||||
t.Fatalf("expected 1 block, got %d", len(blocks))
|
||||
}
|
||||
if blocks[0]["cache_control"] == nil {
|
||||
t.Error("block[0] missing cache_control")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapSystemForDashScopeCache_WithBoundary(t *testing.T) {
|
||||
msg := map[string]any{
|
||||
"role": "system",
|
||||
"content": "Stable prefix\n" + CacheBoundaryMarker + "\nDynamic suffix",
|
||||
}
|
||||
out := wrapSystemForDashScopeCache(msg)
|
||||
blocks := out["content"].([]map[string]any)
|
||||
if len(blocks) != 2 {
|
||||
t.Fatalf("expected 2 blocks, got %d", len(blocks))
|
||||
}
|
||||
if blocks[0]["cache_control"] == nil {
|
||||
t.Error("stable block missing cache_control")
|
||||
}
|
||||
if blocks[1]["cache_control"] != nil {
|
||||
t.Error("dynamic block should not have cache_control")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapSystemForDashScopeCache_NonSystemUntouched(t *testing.T) {
|
||||
msg := map[string]any{
|
||||
"role": "user",
|
||||
"content": "Hello",
|
||||
}
|
||||
out := wrapSystemForDashScopeCache(msg)
|
||||
if !reflect.DeepEqual(out, msg) {
|
||||
t.Error("user message should pass through unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapSystemForDashScopeCache_NonStringContentUntouched(t *testing.T) {
|
||||
blocks := []map[string]any{{"type": "text", "text": "x", "cache_control": map[string]any{"type": "ephemeral"}}}
|
||||
msg := map[string]any{"role": "system", "content": blocks}
|
||||
out := wrapSystemForDashScopeCache(msg)
|
||||
got, _ := json.Marshal(out["content"])
|
||||
want, _ := json.Marshal(blocks)
|
||||
if string(got) != string(want) {
|
||||
t.Errorf("idempotent fail: got %s want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequestBody_DashScopeEndpoint_WrapsSystem(t *testing.T) {
|
||||
p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus")
|
||||
req := ChatRequest{
|
||||
Messages: []Message{
|
||||
{Role: "system", Content: "You are an assistant."},
|
||||
{Role: "user", Content: "Hi"},
|
||||
},
|
||||
}
|
||||
body := p.buildRequestBody("qwen3.6-plus", req, false)
|
||||
msgs := body["messages"].([]map[string]any)
|
||||
sysContent := msgs[0]["content"]
|
||||
if _, ok := sysContent.([]map[string]any); !ok {
|
||||
t.Fatalf("expected DashScope system content as []block, got %T", sysContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequestBody_OpenAINative_DoesNotWrap(t *testing.T) {
|
||||
p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4o")
|
||||
req := ChatRequest{Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}}
|
||||
body := p.buildRequestBody("gpt-4o", req, false)
|
||||
msgs := body["messages"].([]map[string]any)
|
||||
if _, ok := msgs[0]["content"].(string); !ok {
|
||||
t.Errorf("OpenAI native should keep string content, got %T", msgs[0]["content"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDashScopeToolPrefixCache_AddMarkerOnLast(t *testing.T) {
|
||||
tools := []map[string]any{
|
||||
{"type": "function", "function": map[string]any{"name": "tool_a"}},
|
||||
{"type": "function", "function": map[string]any{"name": "tool_b"}},
|
||||
{"type": "function", "function": map[string]any{"name": "tool_c"}},
|
||||
}
|
||||
out := applyDashScopeToolPrefixCache(tools, 1)
|
||||
if len(out) != 3 {
|
||||
t.Fatalf("len changed: got %d", len(out))
|
||||
}
|
||||
if out[0]["cache_control"] != nil || out[1]["cache_control"] != nil {
|
||||
t.Error("non-last tools should not have cache_control")
|
||||
}
|
||||
if out[2]["cache_control"] == nil {
|
||||
t.Error("last tool missing cache_control")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDashScopeToolPrefixCache_EmptyArray(t *testing.T) {
|
||||
out := applyDashScopeToolPrefixCache([]map[string]any{}, 0)
|
||||
if len(out) != 0 {
|
||||
t.Errorf("expected empty, got %d", len(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDashScopeToolPrefixCache_RespectsMarkerLimit(t *testing.T) {
|
||||
tools := []map[string]any{{"type": "function", "function": map[string]any{"name": "x"}}}
|
||||
out := applyDashScopeToolPrefixCache(tools, 4)
|
||||
if out[0]["cache_control"] != nil {
|
||||
t.Error("should skip tool marker when limit reached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountCacheControlMarkers(t *testing.T) {
|
||||
msg := map[string]any{
|
||||
"role": "system",
|
||||
"content": []map[string]any{
|
||||
{"type": "text", "text": "x", "cache_control": map[string]any{"type": "ephemeral"}},
|
||||
{"type": "text", "text": "y"},
|
||||
},
|
||||
}
|
||||
if got := countCacheControlMarkers(msg); got != 1 {
|
||||
t.Errorf("got %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequestBody_DashScopeWithTools_AppliesToolCache(t *testing.T) {
|
||||
p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus")
|
||||
req := ChatRequest{
|
||||
Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}},
|
||||
Tools: []ToolDefinition{
|
||||
{Type: "function", Function: &ToolFunctionSchema{Name: "search", Description: "search docs", Parameters: map[string]any{}}},
|
||||
{Type: "function", Function: &ToolFunctionSchema{Name: "fetch", Description: "fetch url", Parameters: map[string]any{}}},
|
||||
},
|
||||
}
|
||||
body := p.buildRequestBody("qwen3.6-plus", req, false)
|
||||
tools := body["tools"].([]map[string]any)
|
||||
if tools[0]["cache_control"] != nil {
|
||||
t.Error("first tool should not have cache_control")
|
||||
}
|
||||
if tools[len(tools)-1]["cache_control"] == nil {
|
||||
t.Error("last tool should have cache_control")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequestBody_OpenAINativeWithTools_NoToolCache(t *testing.T) {
|
||||
p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4o")
|
||||
req := ChatRequest{
|
||||
Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}},
|
||||
Tools: []ToolDefinition{
|
||||
{Type: "function", Function: &ToolFunctionSchema{Name: "x", Description: "y", Parameters: map[string]any{}}},
|
||||
},
|
||||
}
|
||||
body := p.buildRequestBody("gpt-4o", req, false)
|
||||
tools, ok := body["tools"].([]map[string]any)
|
||||
if ok && len(tools) > 0 && tools[0]["cache_control"] != nil {
|
||||
t.Error("OpenAI native should not have tool cache_control")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequestBody_DashScopeWithEnvDisable_DoesNotWrap(t *testing.T) {
|
||||
t.Setenv("GOCLAW_DISABLE_DASHSCOPE_CACHE", "true")
|
||||
p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus")
|
||||
req := ChatRequest{Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}}
|
||||
body := p.buildRequestBody("qwen3.6-plus", req, false)
|
||||
msgs := body["messages"].([]map[string]any)
|
||||
if _, ok := msgs[0]["content"].(string); !ok {
|
||||
t.Errorf("env disable should keep string content, got %T", msgs[0]["content"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequestBody_BailianThinkingLevelMapsToDashScopeKeys(t *testing.T) {
|
||||
p := NewOpenAIProvider("qwen-richard", "key", "https://proxy.internal/v1", "qwen3.6-plus").
|
||||
WithProviderType("bailian")
|
||||
req := ChatRequest{
|
||||
Model: "qwen3.6-plus",
|
||||
Messages: []Message{
|
||||
{Role: "system", Content: "You are an assistant."},
|
||||
{Role: "user", Content: "Hi"},
|
||||
},
|
||||
Options: map[string]any{OptThinkingLevel: "medium"},
|
||||
}
|
||||
|
||||
body := p.buildRequestBody("qwen3.6-plus", req, false)
|
||||
|
||||
if body[OptEnableThinking] != true {
|
||||
t.Fatalf("enable_thinking = %v, want true", body[OptEnableThinking])
|
||||
}
|
||||
if body[OptThinkingBudget] == nil {
|
||||
t.Fatal("thinking_budget missing for Bailian Qwen thinking model")
|
||||
}
|
||||
if _, has := body[OptReasoningEffort]; has {
|
||||
t.Fatal("Bailian must not receive OpenAI reasoning_effort")
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,8 @@ func TestDashScopeModelSupportsThinking(t *testing.T) {
|
||||
model string
|
||||
want bool
|
||||
}{
|
||||
{"qwen3.7-plus", true},
|
||||
{"qwen3.6-plus", true},
|
||||
{"qwen3.5-plus", true},
|
||||
{"qwen3.5-turbo", true},
|
||||
{"qwen3-max", true},
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenAIUsage_DashScopeCacheHit_Unmarshal(t *testing.T) {
|
||||
raw := `{
|
||||
"prompt_tokens": 2318,
|
||||
"completion_tokens": 195,
|
||||
"total_tokens": 2513,
|
||||
"prompt_tokens_details": {
|
||||
"text_tokens": 2318,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cached_tokens": 2304
|
||||
}
|
||||
}`
|
||||
var u openAIUsage
|
||||
if err := json.Unmarshal([]byte(raw), &u); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if u.PromptTokensDetails.CachedTokens != 2304 {
|
||||
t.Errorf("cached_tokens: got %d want 2304", u.PromptTokensDetails.CachedTokens)
|
||||
}
|
||||
if u.PromptTokensDetails.CacheCreationInputTokens != 0 {
|
||||
t.Errorf("cache_creation: got %d want 0", u.PromptTokensDetails.CacheCreationInputTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIUsage_DashScopeCacheCreate_Unmarshal(t *testing.T) {
|
||||
raw := `{
|
||||
"prompt_tokens": 2318,
|
||||
"prompt_tokens_details": {
|
||||
"cache_creation_input_tokens": 2304,
|
||||
"cached_tokens": 0
|
||||
}
|
||||
}`
|
||||
var u openAIUsage
|
||||
if err := json.Unmarshal([]byte(raw), &u); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if u.PromptTokensDetails.CacheCreationInputTokens != 2304 {
|
||||
t.Errorf("got %d want 2304", u.PromptTokensDetails.CacheCreationInputTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIUsage_NoDetails_OK(t *testing.T) {
|
||||
raw := `{"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}`
|
||||
var u openAIUsage
|
||||
if err := json.Unmarshal([]byte(raw), &u); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if u.PromptTokensDetails != nil {
|
||||
t.Errorf("expected nil PromptTokensDetails, got %+v", u.PromptTokensDetails)
|
||||
}
|
||||
}
|
||||
@@ -112,7 +112,7 @@ func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChun
|
||||
}
|
||||
if chunk.Usage.PromptTokensDetails != nil {
|
||||
result.Usage.CacheReadTokens = chunk.Usage.PromptTokensDetails.CachedTokens
|
||||
result.Usage.CacheCreationTokens = chunk.Usage.PromptTokensDetails.CacheWriteTokens
|
||||
result.Usage.CacheCreationTokens = chunk.Usage.PromptTokensDetails.CacheWriteTokens + chunk.Usage.PromptTokensDetails.CacheCreationInputTokens
|
||||
result.Usage.PromptTokensIncludeCachedSegments = true
|
||||
}
|
||||
if chunk.Usage.CompletionTokensDetails != nil && chunk.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
|
||||
|
||||
@@ -41,15 +41,29 @@ func isDashScopeAPIBase(apiBase string) bool {
|
||||
}
|
||||
|
||||
// dashScopePassthroughKeys is true when enable_thinking / thinking_budget may be added to the JSON body.
|
||||
// Uses URL, provider_type, and name so httptest DashScope URLs still work in tests.
|
||||
// Uses the same DashScope/Bailian route detection as prompt-cache wrapping.
|
||||
func (p *OpenAIProvider) dashScopePassthroughKeys() bool {
|
||||
return p.isDashScope()
|
||||
}
|
||||
|
||||
// isDashScope returns true when this provider routes requests to DashScope/Bailian
|
||||
// (supports cache_control:ephemeral wire format - verified live 2026-05-08).
|
||||
// Uses 3-source detection (URL + providerType + name) to handle reverse-proxied
|
||||
// DashScope endpoints. Includes "bailian" because live qwen-richard provider has
|
||||
// provider_type=bailian.
|
||||
//
|
||||
// Used by buildRequestBody to wrap system content with Anthropic-style
|
||||
// cache_control blocks for prompt caching (90% discount on cached prefix tokens).
|
||||
func (p *OpenAIProvider) isDashScope() bool {
|
||||
if isDashScopeAPIBase(p.apiBase) {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(strings.ToLower(strings.TrimSpace(p.providerType)), "dashscope") {
|
||||
pt := strings.ToLower(strings.TrimSpace(p.providerType))
|
||||
if strings.Contains(pt, "dashscope") || strings.Contains(pt, "bailian") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(strings.ToLower(p.name), "dashscope") {
|
||||
name := strings.ToLower(p.name)
|
||||
if strings.Contains(name, "dashscope") || strings.Contains(name, "bailian") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package providers
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestOpenAIProvider_isDashScope(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
apiBase string
|
||||
ptype string
|
||||
pname string
|
||||
want bool
|
||||
}{
|
||||
{"coding-intl URL", "https://coding-intl.dashscope.aliyuncs.com/v1", "openai_compat", "qwen-richard", true},
|
||||
{"providerType=bailian", "https://custom-proxy.example.com/v1", "bailian", "internal-qwen", true},
|
||||
{"providerType=dashscope", "https://proxy.example.com/v1", "dashscope", "x", true},
|
||||
{"name contains dashscope", "https://proxy.com/v1", "openai_compat", "my-dashscope-mirror", true},
|
||||
{"name contains bailian", "https://proxy.com/v1", "openai_compat", "company-bailian-relay", true},
|
||||
{"openai native", "https://api.openai.com/v1", "openai", "gpt", false},
|
||||
{"anthropic", "https://api.anthropic.com", "anthropic", "claude", false},
|
||||
{"openrouter", "https://openrouter.ai/api/v1", "openai_compat", "openrouter", false},
|
||||
{"empty", "", "", "", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
p := &OpenAIProvider{apiBase: tc.apiBase, providerType: tc.ptype, name: tc.pname}
|
||||
if got := p.isDashScope(); got != tc.want {
|
||||
t.Errorf("isDashScope() = %v, want %v (apiBase=%q ptype=%q name=%q)",
|
||||
got, tc.want, tc.apiBase, tc.ptype, tc.pname)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIProvider_dashScopePassthroughKeys(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
apiBase string
|
||||
ptype string
|
||||
pname string
|
||||
want bool
|
||||
}{
|
||||
{"dashscope URL", "https://coding-intl.dashscope.aliyuncs.com/v1", "openai_compat", "qwen", true},
|
||||
{"providerType=dashscope", "https://proxy.example.com/v1", "dashscope", "qwen", true},
|
||||
{"providerType=bailian", "https://proxy.example.com/v1", "bailian", "qwen", true},
|
||||
{"name contains dashscope", "https://proxy.example.com/v1", "openai_compat", "team-dashscope", true},
|
||||
{"name contains bailian", "https://proxy.example.com/v1", "openai_compat", "team-bailian", true},
|
||||
{"openrouter", "https://openrouter.ai/api/v1", "openai_compat", "openrouter", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
p := &OpenAIProvider{apiBase: tc.apiBase, providerType: tc.ptype, name: tc.pname}
|
||||
if got := p.dashScopePassthroughKeys(); got != tc.want {
|
||||
t.Errorf("dashScopePassthroughKeys() = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -134,7 +134,7 @@ func (p *OpenAIProvider) parseResponse(resp *openAIResponse) *ChatResponse {
|
||||
}
|
||||
if resp.Usage.PromptTokensDetails != nil {
|
||||
result.Usage.CacheReadTokens = resp.Usage.PromptTokensDetails.CachedTokens
|
||||
result.Usage.CacheCreationTokens = resp.Usage.PromptTokensDetails.CacheWriteTokens
|
||||
result.Usage.CacheCreationTokens = resp.Usage.PromptTokensDetails.CacheWriteTokens + resp.Usage.PromptTokensDetails.CacheCreationInputTokens
|
||||
result.Usage.PromptTokensIncludeCachedSegments = true
|
||||
}
|
||||
if resp.Usage.CompletionTokensDetails != nil && resp.Usage.CompletionTokensDetails.ReasoningTokens > 0 {
|
||||
|
||||
@@ -159,6 +159,15 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream
|
||||
msgs = append(msgs, msg)
|
||||
}
|
||||
|
||||
// Apply DashScope cache_control wrapping (verified live 2026-05-08).
|
||||
// Uses 3-source detection from p.isDashScope() (URL + providerType + name)
|
||||
// to handle reverse-proxied endpoints. No-op for non-DashScope endpoints
|
||||
// or when env disabled. For native OpenAI, role mapping above renames
|
||||
// "system"→"developer" so wrap is a no-op (role guard).
|
||||
if p.isDashScope() && !dashScopeCacheDisabled() && len(msgs) > 0 {
|
||||
msgs[0] = wrapSystemForDashScopeCache(msgs[0])
|
||||
}
|
||||
|
||||
// Safety net: strip trailing assistant message to prevent HTTP 400 from
|
||||
// proxy providers (LiteLLM, OpenRouter) that don't support assistant prefill.
|
||||
// This should rarely trigger — the agent loop ensures user message is last.
|
||||
@@ -185,6 +194,19 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream
|
||||
}
|
||||
}
|
||||
|
||||
// DashScope tool prefix cache: cache_control on last tool definition
|
||||
// caches the entire tools array (descriptions + schemas, ~5-10K tokens).
|
||||
// Combined with system block cache: 2/4 markers used, 99.5% hit rate verified.
|
||||
if p.isDashScope() && !dashScopeCacheDisabled() {
|
||||
if t, ok := body["tools"].([]map[string]any); ok && len(t) > 0 {
|
||||
markersFromSystem := 0
|
||||
if len(msgs) > 0 {
|
||||
markersFromSystem = countCacheControlMarkers(msgs[0])
|
||||
}
|
||||
body["tools"] = applyDashScopeToolPrefixCache(t, markersFromSystem)
|
||||
}
|
||||
}
|
||||
|
||||
// Together returns HTTP 400 on some requests when stream_options is present.
|
||||
if stream && !p.isTogetherEndpoint() {
|
||||
body["stream_options"] = map[string]any{
|
||||
@@ -249,6 +271,10 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream
|
||||
|
||||
// DashScope-specific passthrough keys — never send to other OpenAI-compat hosts.
|
||||
if p.dashScopePassthroughKeys() {
|
||||
if level, ok := req.Options[OptThinkingLevel].(string); ok && level != "" && level != "off" && dashscopeThinkingModels[model] {
|
||||
body[OptEnableThinking] = true
|
||||
body[OptThinkingBudget] = dashscopeThinkingBudget(level)
|
||||
}
|
||||
if v, ok := req.Options[OptEnableThinking]; ok {
|
||||
body[OptEnableThinking] = v
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
package providers
|
||||
|
||||
// BuildRequestBodyForTest is a test-only export of buildRequestBody for
|
||||
// integration smoke tests in the tests/integration package. NOT part of the
|
||||
// public API - do not call from production code paths.
|
||||
func (p *OpenAIProvider) BuildRequestBodyForTest(model string, req ChatRequest, stream bool) map[string]any {
|
||||
return p.buildRequestBody(model, req, stream)
|
||||
}
|
||||
@@ -56,8 +56,9 @@ type openAIUsage struct {
|
||||
}
|
||||
|
||||
type openAIPromptDetails struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens,omitempty"` // some OpenAI-compat providers
|
||||
CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` // DashScope/Qwen
|
||||
}
|
||||
|
||||
type openAICompletionDetails struct {
|
||||
|
||||
Reference in new issue
Block a user