feat: Add native Gemini image generation support and refine media path stripping in agent output.

This commit is contained in:
viettranx committed 2026-02-28 18:44:51 +07:00
1 parent 50a90aa8c6
commit 45ea0ee9a4
2 files changed
+109 -3

No files matched your search

+7 -1
View File
@@ -293,7 +293,13 @@ func stripMediaPaths(content string) string {
var result []string
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "MEDIA:") || strings.HasPrefix(trimmed, "[[audio_as_voice]]") {
if strings.HasPrefix(trimmed, "[[audio_as_voice]]") {
continue
}
// Strip any line containing a MEDIA: path reference, regardless of wrapping format.
// LLMs echo these in many forms: bare "MEDIA:/path", markdown "![alt](MEDIA:/path)",
// JSON '{"image":"MEDIA:/path"}', etc. The /tmp/ or / after MEDIA: confirms it's a path.
if strings.Contains(trimmed, "MEDIA:/") {
continue
}
result = append(result, line)
+102 -2
View File
@@ -92,10 +92,17 @@ func (t *CreateImageTool) Execute(ctx context.Context, args map[string]interface
slog.Info("create_image: calling image generation API",
"provider", providerName, "model", model, "aspect_ratio", aspectRatio)
// OpenRouter supports modalities on /chat/completions; Gemini, OpenAI and others use /images/generations
// Route to the correct image generation endpoint per provider:
// - gemini: native Gemini generateContent API (responseModalities)
// - openrouter: OpenAI-compat /chat/completions with modalities
// - others (openai, etc.): /images/generations
var imageBytes []byte
var usage *providers.Usage
if providerName == "openrouter" {
if providerName == "gemini" {
var genErr error
imageBytes, usage, genErr = t.callGeminiNativeImageGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt)
err = genErr
} else if providerName == "openrouter" {
var genErr error
imageBytes, usage, genErr = t.callImageGenAPI(ctx, cp.APIKey(), cp.APIBase(), model, prompt, aspectRatio)
err = genErr
@@ -290,6 +297,99 @@ func (t *CreateImageTool) callStandardImageGenAPI(ctx context.Context, apiKey, a
return imageBytes, nil, nil
}
// callGeminiNativeImageGen uses the native Gemini generateContent API with responseModalities.
// Gemini image models (gemini-2.5-flash-image, gemini-3.1-flash-image-preview) require this
// endpoint — they don't support the OpenAI-compat /images/generations or /chat/completions.
func (t *CreateImageTool) callGeminiNativeImageGen(ctx context.Context, apiKey, apiBase, model, prompt string) ([]byte, *providers.Usage, error) {
// Derive native Gemini base from OpenAI-compat base (strip /openai suffix)
nativeBase := strings.TrimRight(apiBase, "/")
nativeBase = strings.TrimSuffix(nativeBase, "/openai")
url := fmt.Sprintf("%s/models/%s:generateContent?key=%s", nativeBase, model, apiKey)
body := map[string]interface{}{
"contents": []map[string]interface{}{
{"parts": []map[string]interface{}{{"text": prompt}}},
},
"generationConfig": map[string]interface{}{
"responseModalities": []string{"TEXT", "IMAGE"},
},
}
jsonBody, err := json.Marshal(body)
if err != nil {
return nil, nil, fmt.Errorf("marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
if err != nil {
return nil, nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: 120 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("http request: %w", err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
}
// Parse native Gemini response: {candidates: [{content: {parts: [{inlineData: {mimeType, data}}]}}]}
var gemResp struct {
Candidates []struct {
Content struct {
Parts []struct {
InlineData *struct {
MimeType string `json:"mimeType"`
Data string `json:"data"`
} `json:"inlineData"`
Text string `json:"text"`
} `json:"parts"`
} `json:"content"`
} `json:"candidates"`
UsageMetadata *struct {
PromptTokenCount int `json:"promptTokenCount"`
CandidatesTokenCount int `json:"candidatesTokenCount"`
TotalTokenCount int `json:"totalTokenCount"`
} `json:"usageMetadata"`
}
if err := json.Unmarshal(respBody, &gemResp); err != nil {
return nil, nil, fmt.Errorf("parse response: %w", err)
}
// Extract first image from parts
for _, cand := range gemResp.Candidates {
for _, part := range cand.Content.Parts {
if part.InlineData != nil && strings.HasPrefix(part.InlineData.MimeType, "image/") {
imageBytes, err := base64.StdEncoding.DecodeString(part.InlineData.Data)
if err != nil {
return nil, nil, fmt.Errorf("decode base64: %w", err)
}
var usage *providers.Usage
if gemResp.UsageMetadata != nil {
usage = &providers.Usage{
PromptTokens: gemResp.UsageMetadata.PromptTokenCount,
CompletionTokens: gemResp.UsageMetadata.CandidatesTokenCount,
TotalTokens: gemResp.UsageMetadata.TotalTokenCount,
}
}
return imageBytes, usage, nil
}
}
}
return nil, nil, fmt.Errorf("no image data in Gemini response")
}
// parseImageResponse extracts base64 image data from the OpenAI-compat chat response.
// Looks for images in choices[0].message.content (multipart) or choices[0].message.images.
func (t *CreateImageTool) parseImageResponse(respBody []byte) ([]byte, *providers.Usage, error) {