mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-13 12:14:40 +00:00
feat: Add native Gemini image generation support and refine media path stripping in agent output.
This commit is contained in:
1 parent
50a90aa8c6
commit
45ea0ee9a4
2 files changed
+109
-3
No files matched your search
@@ -293,7 +293,13 @@ func stripMediaPaths(content string) string {
|
||||
var result []string
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "MEDIA:") || strings.HasPrefix(trimmed, "[[audio_as_voice]]") {
|
||||
if strings.HasPrefix(trimmed, "[[audio_as_voice]]") {
|
||||
continue
|
||||
}
|
||||
// Strip any line containing a MEDIA: path reference, regardless of wrapping format.
|
||||
// LLMs echo these in many forms: bare "MEDIA:/path", markdown "",
|
||||
// JSON '{"image":"MEDIA:/path"}', etc. The /tmp/ or / after MEDIA: confirms it's a path.
|
||||
if strings.Contains(trimmed, "MEDIA:/") {
|
||||
continue
|
||||
}
|
||||
result = append(result, line)
|
||||
|
||||
@@ -92,10 +92,17 @@ func (t *CreateImageTool) Execute(ctx context.Context, args map[string]interface
|
||||
slog.Info("create_image: calling image generation API",
|
||||
"provider", providerName, "model", model, "aspect_ratio", aspectRatio)
|
||||
|
||||
// OpenRouter supports modalities on /chat/completions; Gemini, OpenAI and others use /images/generations
|
||||
// Route to the correct image generation endpoint per provider:
|
||||
// - gemini: native Gemini generateContent API (responseModalities)
|
||||
// - openrouter: OpenAI-compat /chat/completions with modalities
|
||||
// - others (openai, etc.): /images/generations
|
||||
var imageBytes []byte
|
||||
var usage *providers.Usage
|
||||
if providerName == "openrouter" {
|
||||
if providerName == "gemini" {
|
||||
var genErr error
|
||||
imageBytes, usage, genErr = t.callGeminiNativeImageGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt)
|
||||
err = genErr
|
||||
} else if providerName == "openrouter" {
|
||||
var genErr error
|
||||
imageBytes, usage, genErr = t.callImageGenAPI(ctx, cp.APIKey(), cp.APIBase(), model, prompt, aspectRatio)
|
||||
err = genErr
|
||||
@@ -290,6 +297,99 @@ func (t *CreateImageTool) callStandardImageGenAPI(ctx context.Context, apiKey, a
|
||||
return imageBytes, nil, nil
|
||||
}
|
||||
|
||||
// callGeminiNativeImageGen uses the native Gemini generateContent API with responseModalities.
|
||||
// Gemini image models (gemini-2.5-flash-image, gemini-3.1-flash-image-preview) require this
|
||||
// endpoint — they don't support the OpenAI-compat /images/generations or /chat/completions.
|
||||
func (t *CreateImageTool) callGeminiNativeImageGen(ctx context.Context, apiKey, apiBase, model, prompt string) ([]byte, *providers.Usage, error) {
|
||||
// Derive native Gemini base from OpenAI-compat base (strip /openai suffix)
|
||||
nativeBase := strings.TrimRight(apiBase, "/")
|
||||
nativeBase = strings.TrimSuffix(nativeBase, "/openai")
|
||||
|
||||
url := fmt.Sprintf("%s/models/%s:generateContent?key=%s", nativeBase, model, apiKey)
|
||||
|
||||
body := map[string]interface{}{
|
||||
"contents": []map[string]interface{}{
|
||||
{"parts": []map[string]interface{}{{"text": prompt}}},
|
||||
},
|
||||
"generationConfig": map[string]interface{}{
|
||||
"responseModalities": []string{"TEXT", "IMAGE"},
|
||||
},
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 120 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("http request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
|
||||
}
|
||||
|
||||
// Parse native Gemini response: {candidates: [{content: {parts: [{inlineData: {mimeType, data}}]}}]}
|
||||
var gemResp struct {
|
||||
Candidates []struct {
|
||||
Content struct {
|
||||
Parts []struct {
|
||||
InlineData *struct {
|
||||
MimeType string `json:"mimeType"`
|
||||
Data string `json:"data"`
|
||||
} `json:"inlineData"`
|
||||
Text string `json:"text"`
|
||||
} `json:"parts"`
|
||||
} `json:"content"`
|
||||
} `json:"candidates"`
|
||||
UsageMetadata *struct {
|
||||
PromptTokenCount int `json:"promptTokenCount"`
|
||||
CandidatesTokenCount int `json:"candidatesTokenCount"`
|
||||
TotalTokenCount int `json:"totalTokenCount"`
|
||||
} `json:"usageMetadata"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &gemResp); err != nil {
|
||||
return nil, nil, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
|
||||
// Extract first image from parts
|
||||
for _, cand := range gemResp.Candidates {
|
||||
for _, part := range cand.Content.Parts {
|
||||
if part.InlineData != nil && strings.HasPrefix(part.InlineData.MimeType, "image/") {
|
||||
imageBytes, err := base64.StdEncoding.DecodeString(part.InlineData.Data)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("decode base64: %w", err)
|
||||
}
|
||||
var usage *providers.Usage
|
||||
if gemResp.UsageMetadata != nil {
|
||||
usage = &providers.Usage{
|
||||
PromptTokens: gemResp.UsageMetadata.PromptTokenCount,
|
||||
CompletionTokens: gemResp.UsageMetadata.CandidatesTokenCount,
|
||||
TotalTokens: gemResp.UsageMetadata.TotalTokenCount,
|
||||
}
|
||||
}
|
||||
return imageBytes, usage, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("no image data in Gemini response")
|
||||
}
|
||||
|
||||
// parseImageResponse extracts base64 image data from the OpenAI-compat chat response.
|
||||
// Looks for images in choices[0].message.content (multipart) or choices[0].message.images.
|
||||
func (t *CreateImageTool) parseImageResponse(respBody []byte) ([]byte, *providers.Usage, error) {
|
||||
|
||||
Reference in new issue
Block a user