fix(tools): correct Veo API, media ref ordering, video tag, and model verify

- Fix create_video: use predictLongRunning API instead of generateContent
  (async polling flow: POST → poll every 10s → download video from URI)
- Fix durationSeconds as int (not string) per actual Gemini API requirement
- Fix MediaRef collection order: historical first, current last, so
  refs[len-1] always returns the most recent file (fixes read_audio
  picking up old file instead of current voice message)
- Remove misleading "video not yet supported" text from Telegram handler
  that prevented LLM from calling read_video tool
- Add isNonChatModel() to skip chat-based verify for generation models
  (veo-*, dall-e-*, imagen-*, gemini-*-image)
This commit is contained in:
viettranx committed 2026-03-08 15:21:08 +07:00
1 parent 691ddce8fb
commit e1a6801a7a
4 files changed
+185 -89

No files matched your search

+19 -18
View File
@@ -341,13 +341,9 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
ctx = l.loadHistoricalImagesForTool(ctx, mediaRefs, messages)
}
// 2b. Collect document MediaRefs (current + historical) for read_document tool.
// 2b. Collect document MediaRefs (historical + current) for read_document tool.
// Historical first, current last — so refs[len-1] is always the most recent file.
var docRefs []providers.MediaRef
for _, ref := range mediaRefs {
if ref.Kind == "document" {
docRefs = append(docRefs, ref)
}
}
for i := len(messages) - 1; i >= 0; i-- {
for _, ref := range messages[i].MediaRefs {
if ref.Kind == "document" {
@@ -355,17 +351,17 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
}
}
}
for _, ref := range mediaRefs {
if ref.Kind == "document" {
docRefs = append(docRefs, ref)
}
}
if len(docRefs) > 0 {
ctx = tools.WithMediaDocRefs(ctx, docRefs)
}
// 2c. Collect audio MediaRefs (current + historical) for read_audio tool.
// 2c. Collect audio MediaRefs (historical + current) for read_audio tool.
var audioRefs []providers.MediaRef
for _, ref := range mediaRefs {
if ref.Kind == "audio" {
audioRefs = append(audioRefs, ref)
}
}
for i := len(messages) - 1; i >= 0; i-- {
for _, ref := range messages[i].MediaRefs {
if ref.Kind == "audio" {
@@ -373,17 +369,17 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
}
}
}
for _, ref := range mediaRefs {
if ref.Kind == "audio" {
audioRefs = append(audioRefs, ref)
}
}
if len(audioRefs) > 0 {
ctx = tools.WithMediaAudioRefs(ctx, audioRefs)
}
// 2d. Collect video MediaRefs (current + historical) for read_video tool.
// 2d. Collect video MediaRefs (historical + current) for read_video tool.
var videoRefs []providers.MediaRef
for _, ref := range mediaRefs {
if ref.Kind == "video" {
videoRefs = append(videoRefs, ref)
}
}
for i := len(messages) - 1; i >= 0; i-- {
for _, ref := range messages[i].MediaRefs {
if ref.Kind == "video" {
@@ -391,6 +387,11 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
}
}
}
for _, ref := range mediaRefs {
if ref.Kind == "video" {
videoRefs = append(videoRefs, ref)
}
}
if len(videoRefs) > 0 {
ctx = tools.WithMediaVideoRefs(ctx, videoRefs)
}
+1 -6
View File
@@ -242,12 +242,7 @@ func (c *Channel) handleMessage(ctx context.Context, update telego.Update) {
}
case "video", "animation":
// Video: notify user that video is not fully supported yet.
// Only add the notice when there is no caption/text — media tags haven't been
// prepended yet at this stage of the pipeline.
if content == "" {
extraContent += "\n\n[Video received — video content analysis is not yet supported, only caption text is processed]"
}
// Video files are handled by the read_video tool via MediaRef pipeline.
}
if m.FilePath != "" {
+24
View File
@@ -64,6 +64,13 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R
return
}
// Non-chat models (image/video generation) can't be verified via Chat API.
// Accept them if the provider is reachable (already validated above).
if isNonChatModel(req.Model) {
writeJSON(w, http.StatusOK, map[string]interface{}{"valid": true, "note": "generation model accepted (not verifiable via chat)"})
return
}
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
@@ -120,6 +127,23 @@ func (h *ProvidersHandler) handleClaudeCLIAuthStatus(w http.ResponseWriter, r *h
})
}
// isNonChatModel returns true for models that cannot be verified via Chat API
// (image/video generation models).
func isNonChatModel(model string) bool {
nonChatPrefixes := []string{
"veo-", "google/veo-",
"dall-e-", "imagen-", "google/imagen-",
"gemini-2.5-flash-image", "google/gemini-2.5-flash-image",
}
m := strings.ToLower(model)
for _, prefix := range nonChatPrefixes {
if strings.HasPrefix(m, prefix) || strings.Contains(m, "/"+prefix) {
return true
}
}
return false
}
// friendlyVerifyError extracts a human-readable message from provider errors.
// Raw errors often contain JSON blobs like: `HTTP 400: minimax: {"type":"error","error":{"type":"bad_request_error","message":"unknown model ..."}}`
func friendlyVerifyError(err error) string {
+141 -65
View File
@@ -3,7 +3,6 @@ package tools
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -28,7 +27,7 @@ var videoGenModelDefaults = map[string]string{
}
// CreateVideoTool generates videos using a video generation API.
// Uses Gemini Veo via native generateContent API with VIDEO response modality.
// Uses Gemini Veo via predictLongRunning API (async with polling).
type CreateVideoTool struct {
registry *providers.Registry
}
@@ -53,11 +52,11 @@ func (t *CreateVideoTool) Parameters() map[string]interface{} {
},
"duration": map[string]interface{}{
"type": "integer",
"description": "Video duration in seconds (default 5, max 30).",
"description": "Video duration in seconds: 4, 6, or 8 (default 8).",
},
"aspect_ratio": map[string]interface{}{
"type": "string",
"description": "Aspect ratio: '16:9' (default), '9:16', '1:1'.",
"description": "Aspect ratio: '16:9' (default) or '9:16'.",
},
},
"required": []string{"prompt"},
@@ -70,26 +69,28 @@ func (t *CreateVideoTool) Execute(ctx context.Context, args map[string]interface
return ErrorResult("prompt is required")
}
// Parse and enforce duration (default 5, max 30).
duration := 5
// Parse and enforce duration. Veo supports 4, 6, or 8 seconds.
duration := 8
if d, ok := args["duration"].(float64); ok {
duration = int(d)
}
if duration < 1 {
duration = 1
}
if duration > 30 {
duration = 30
switch {
case duration <= 4:
duration = 4
case duration <= 6:
duration = 6
default:
duration = 8
}
// Parse and validate aspect ratio.
// Parse and validate aspect ratio. Veo supports 16:9 and 9:16.
aspectRatio := "16:9"
if ar, _ := args["aspect_ratio"].(string); ar != "" {
switch ar {
case "16:9", "9:16", "1:1":
case "16:9", "9:16":
aspectRatio = ar
default:
return ErrorResult(fmt.Sprintf("unsupported aspect_ratio %q; use '16:9', '9:16', or '1:1'", ar))
return ErrorResult(fmt.Sprintf("unsupported aspect_ratio %q; use '16:9' or '9:16'", ar))
}
}
@@ -186,24 +187,24 @@ func (t *CreateVideoTool) resolveConfig(ctx context.Context) (providerName, mode
return providerName, model
}
// callGeminiVideoGen uses the native Gemini generateContent API with VIDEO response modality.
// callGeminiVideoGen uses the Gemini predictLongRunning API for Veo video generation.
// Flow: POST predictLongRunning → poll operation → download video from URI.
func (t *CreateVideoTool) callGeminiVideoGen(ctx context.Context, apiKey, apiBase, model, prompt string, duration int, aspectRatio string) ([]byte, *providers.Usage, error) {
nativeBase := strings.TrimRight(apiBase, "/")
nativeBase = strings.TrimSuffix(nativeBase, "/openai")
url := fmt.Sprintf("%s/models/%s:generateContent?key=%s", nativeBase, model, apiKey)
genConfig := map[string]interface{}{
"responseModalities": []string{"VIDEO"},
"videoDuration": duration,
"aspectRatio": aspectRatio,
}
// 1. Start long-running video generation.
predictURL := fmt.Sprintf("%s/models/%s:predictLongRunning", nativeBase, model)
body := map[string]interface{}{
"contents": []map[string]interface{}{
{"parts": []map[string]interface{}{{"text": prompt}}},
"instances": []map[string]interface{}{
{"prompt": prompt},
},
"parameters": map[string]interface{}{
"aspectRatio": aspectRatio,
"durationSeconds": duration,
"personGeneration": "allow_all",
},
"generationConfig": genConfig,
}
jsonBody, err := json.Marshal(body)
@@ -211,14 +212,14 @@ func (t *CreateVideoTool) callGeminiVideoGen(ctx context.Context, apiKey, apiBas
return nil, nil, fmt.Errorf("marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
req, err := http.NewRequestWithContext(ctx, "POST", predictURL, bytes.NewReader(jsonBody))
if err != nil {
return nil, nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("x-goog-api-key", apiKey)
// Video generation can take a while.
client := &http.Client{Timeout: 300 * time.Second}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("http request: %w", err)
@@ -233,49 +234,124 @@ func (t *CreateVideoTool) callGeminiVideoGen(ctx context.Context, apiKey, apiBas
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
}
// Parse Gemini response — look for inlineData with video MIME.
var gemResp struct {
Candidates []struct {
Content struct {
Parts []struct {
InlineData *struct {
MimeType string `json:"mimeType"`
Data string `json:"data"`
} `json:"inlineData"`
} `json:"parts"`
} `json:"content"`
} `json:"candidates"`
UsageMetadata *struct {
PromptTokenCount int `json:"promptTokenCount"`
CandidatesTokenCount int `json:"candidatesTokenCount"`
TotalTokenCount int `json:"totalTokenCount"`
} `json:"usageMetadata"`
var opResp struct {
Name string `json:"name"`
Done bool `json:"done"`
}
if err := json.Unmarshal(respBody, &gemResp); err != nil {
return nil, nil, fmt.Errorf("parse response: %w", err)
if err := json.Unmarshal(respBody, &opResp); err != nil {
return nil, nil, fmt.Errorf("parse operation response: %w", err)
}
if opResp.Name == "" {
return nil, nil, fmt.Errorf("no operation name in response: %s", truncateBytes(respBody, 300))
}
for _, cand := range gemResp.Candidates {
for _, part := range cand.Content.Parts {
if part.InlineData != nil && strings.HasPrefix(part.InlineData.MimeType, "video/") {
videoBytes, err := base64.StdEncoding.DecodeString(part.InlineData.Data)
if err != nil {
return nil, nil, fmt.Errorf("decode base64: %w", err)
}
var usage *providers.Usage
if gemResp.UsageMetadata != nil {
usage = &providers.Usage{
PromptTokens: gemResp.UsageMetadata.PromptTokenCount,
CompletionTokens: gemResp.UsageMetadata.CandidatesTokenCount,
TotalTokens: gemResp.UsageMetadata.TotalTokenCount,
}
}
return videoBytes, usage, nil
}
slog.Info("create_video: operation started", "operation", opResp.Name)
// 2. Poll operation until done (max ~6 minutes, poll every 10s).
pollURL := fmt.Sprintf("%s/%s", nativeBase, opResp.Name)
const maxPolls = 40
const pollInterval = 10 * time.Second
var doneBody []byte
for i := 0; i < maxPolls; i++ {
select {
case <-ctx.Done():
return nil, nil, ctx.Err()
case <-time.After(pollInterval):
}
pollReq, err := http.NewRequestWithContext(ctx, "GET", pollURL, nil)
if err != nil {
return nil, nil, fmt.Errorf("create poll request: %w", err)
}
pollReq.Header.Set("x-goog-api-key", apiKey)
pollResp, err := client.Do(pollReq)
if err != nil {
slog.Warn("create_video: poll error, retrying", "error", err, "attempt", i+1)
continue
}
pollBody, _ := io.ReadAll(pollResp.Body)
pollResp.Body.Close()
if pollResp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("poll API error %d: %s", pollResp.StatusCode, truncateBytes(pollBody, 500))
}
var pollOp struct {
Done bool `json:"done"`
Error json.RawMessage `json:"error"`
}
if err := json.Unmarshal(pollBody, &pollOp); err != nil {
return nil, nil, fmt.Errorf("parse poll response: %w", err)
}
if pollOp.Error != nil && string(pollOp.Error) != "null" {
return nil, nil, fmt.Errorf("video generation error: %s", truncateBytes(pollOp.Error, 500))
}
if pollOp.Done {
doneBody = pollBody
break
}
slog.Info("create_video: polling", "attempt", i+1, "done", pollOp.Done)
}
return nil, nil, fmt.Errorf("no video data in Gemini response")
if doneBody == nil {
return nil, nil, fmt.Errorf("video generation timed out after %d polls", maxPolls)
}
// 3. Extract video URI and download.
var result struct {
Response struct {
GenerateVideoResponse struct {
GeneratedSamples []struct {
Video struct {
URI string `json:"uri"`
MimeType string `json:"mimeType"`
} `json:"video"`
} `json:"generatedSamples"`
} `json:"generateVideoResponse"`
} `json:"response"`
}
if err := json.Unmarshal(doneBody, &result); err != nil {
return nil, nil, fmt.Errorf("parse final response: %w", err)
}
samples := result.Response.GenerateVideoResponse.GeneratedSamples
if len(samples) == 0 || samples[0].Video.URI == "" {
return nil, nil, fmt.Errorf("no video in response: %s", truncateBytes(doneBody, 300))
}
videoURI := samples[0].Video.URI
slog.Info("create_video: downloading video", "uri", videoURI)
dlReq, err := http.NewRequestWithContext(ctx, "GET", videoURI, nil)
if err != nil {
return nil, nil, fmt.Errorf("create download request: %w", err)
}
dlReq.Header.Set("x-goog-api-key", apiKey)
dlClient := &http.Client{Timeout: 120 * time.Second}
dlResp, err := dlClient.Do(dlReq)
if err != nil {
return nil, nil, fmt.Errorf("download video: %w", err)
}
defer dlResp.Body.Close()
if dlResp.StatusCode != http.StatusOK {
dlBody, _ := io.ReadAll(dlResp.Body)
return nil, nil, fmt.Errorf("download error %d: %s", dlResp.StatusCode, truncateBytes(dlBody, 300))
}
videoBytes, err := io.ReadAll(dlResp.Body)
if err != nil {
return nil, nil, fmt.Errorf("read video data: %w", err)
}
return videoBytes, nil, nil
}
// callChatVideoGen tries OpenAI-compatible chat completions with video modality.