diff --git a/internal/agent/loop.go b/internal/agent/loop.go index 9e9ade3b..dcb86381 100644 --- a/internal/agent/loop.go +++ b/internal/agent/loop.go @@ -341,13 +341,9 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error) ctx = l.loadHistoricalImagesForTool(ctx, mediaRefs, messages) } - // 2b. Collect document MediaRefs (current + historical) for read_document tool. + // 2b. Collect document MediaRefs (historical + current) for read_document tool. + // Historical first, current last — so refs[len-1] is always the most recent file. var docRefs []providers.MediaRef - for _, ref := range mediaRefs { - if ref.Kind == "document" { - docRefs = append(docRefs, ref) - } - } for i := len(messages) - 1; i >= 0; i-- { for _, ref := range messages[i].MediaRefs { if ref.Kind == "document" { @@ -355,17 +351,17 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error) } } } + for _, ref := range mediaRefs { + if ref.Kind == "document" { + docRefs = append(docRefs, ref) + } + } if len(docRefs) > 0 { ctx = tools.WithMediaDocRefs(ctx, docRefs) } - // 2c. Collect audio MediaRefs (current + historical) for read_audio tool. + // 2c. Collect audio MediaRefs (historical + current) for read_audio tool. var audioRefs []providers.MediaRef - for _, ref := range mediaRefs { - if ref.Kind == "audio" { - audioRefs = append(audioRefs, ref) - } - } for i := len(messages) - 1; i >= 0; i-- { for _, ref := range messages[i].MediaRefs { if ref.Kind == "audio" { @@ -373,17 +369,17 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error) } } } + for _, ref := range mediaRefs { + if ref.Kind == "audio" { + audioRefs = append(audioRefs, ref) + } + } if len(audioRefs) > 0 { ctx = tools.WithMediaAudioRefs(ctx, audioRefs) } - // 2d. Collect video MediaRefs (current + historical) for read_video tool. + // 2d. Collect video MediaRefs (historical + current) for read_video tool. var videoRefs []providers.MediaRef - for _, ref := range mediaRefs { - if ref.Kind == "video" { - videoRefs = append(videoRefs, ref) - } - } for i := len(messages) - 1; i >= 0; i-- { for _, ref := range messages[i].MediaRefs { if ref.Kind == "video" { @@ -391,6 +387,11 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error) } } } + for _, ref := range mediaRefs { + if ref.Kind == "video" { + videoRefs = append(videoRefs, ref) + } + } if len(videoRefs) > 0 { ctx = tools.WithMediaVideoRefs(ctx, videoRefs) } diff --git a/internal/channels/telegram/handlers.go b/internal/channels/telegram/handlers.go index 414b7081..1f56ec21 100644 --- a/internal/channels/telegram/handlers.go +++ b/internal/channels/telegram/handlers.go @@ -242,12 +242,7 @@ func (c *Channel) handleMessage(ctx context.Context, update telego.Update) { } case "video", "animation": - // Video: notify user that video is not fully supported yet. - // Only add the notice when there is no caption/text — media tags haven't been - // prepended yet at this stage of the pipeline. - if content == "" { - extraContent += "\n\n[Video received — video content analysis is not yet supported, only caption text is processed]" - } + // Video files are handled by the read_video tool via MediaRef pipeline. } if m.FilePath != "" { diff --git a/internal/http/provider_verify.go b/internal/http/provider_verify.go index 20cccba3..d91d2911 100644 --- a/internal/http/provider_verify.go +++ b/internal/http/provider_verify.go @@ -64,6 +64,13 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R return } + // Non-chat models (image/video generation) can't be verified via Chat API. + // Accept them if the provider is reachable (already validated above). + if isNonChatModel(req.Model) { + writeJSON(w, http.StatusOK, map[string]interface{}{"valid": true, "note": "generation model accepted (not verifiable via chat)"}) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) defer cancel() @@ -120,6 +127,23 @@ func (h *ProvidersHandler) handleClaudeCLIAuthStatus(w http.ResponseWriter, r *h }) } +// isNonChatModel returns true for models that cannot be verified via Chat API +// (image/video generation models). +func isNonChatModel(model string) bool { + nonChatPrefixes := []string{ + "veo-", "google/veo-", + "dall-e-", "imagen-", "google/imagen-", + "gemini-2.5-flash-image", "google/gemini-2.5-flash-image", + } + m := strings.ToLower(model) + for _, prefix := range nonChatPrefixes { + if strings.HasPrefix(m, prefix) || strings.Contains(m, "/"+prefix) { + return true + } + } + return false +} + // friendlyVerifyError extracts a human-readable message from provider errors. // Raw errors often contain JSON blobs like: `HTTP 400: minimax: {"type":"error","error":{"type":"bad_request_error","message":"unknown model ..."}}` func friendlyVerifyError(err error) string { diff --git a/internal/tools/create_video.go b/internal/tools/create_video.go index a1b9a318..bec782d3 100644 --- a/internal/tools/create_video.go +++ b/internal/tools/create_video.go @@ -3,7 +3,6 @@ package tools import ( "bytes" "context" - "encoding/base64" "encoding/json" "fmt" "io" @@ -28,7 +27,7 @@ var videoGenModelDefaults = map[string]string{ } // CreateVideoTool generates videos using a video generation API. -// Uses Gemini Veo via native generateContent API with VIDEO response modality. +// Uses Gemini Veo via predictLongRunning API (async with polling). type CreateVideoTool struct { registry *providers.Registry } @@ -53,11 +52,11 @@ func (t *CreateVideoTool) Parameters() map[string]interface{} { }, "duration": map[string]interface{}{ "type": "integer", - "description": "Video duration in seconds (default 5, max 30).", + "description": "Video duration in seconds: 4, 6, or 8 (default 8).", }, "aspect_ratio": map[string]interface{}{ "type": "string", - "description": "Aspect ratio: '16:9' (default), '9:16', '1:1'.", + "description": "Aspect ratio: '16:9' (default) or '9:16'.", }, }, "required": []string{"prompt"}, @@ -70,26 +69,28 @@ func (t *CreateVideoTool) Execute(ctx context.Context, args map[string]interface return ErrorResult("prompt is required") } - // Parse and enforce duration (default 5, max 30). - duration := 5 + // Parse and enforce duration. Veo supports 4, 6, or 8 seconds. + duration := 8 if d, ok := args["duration"].(float64); ok { duration = int(d) } - if duration < 1 { - duration = 1 - } - if duration > 30 { - duration = 30 + switch { + case duration <= 4: + duration = 4 + case duration <= 6: + duration = 6 + default: + duration = 8 } - // Parse and validate aspect ratio. + // Parse and validate aspect ratio. Veo supports 16:9 and 9:16. aspectRatio := "16:9" if ar, _ := args["aspect_ratio"].(string); ar != "" { switch ar { - case "16:9", "9:16", "1:1": + case "16:9", "9:16": aspectRatio = ar default: - return ErrorResult(fmt.Sprintf("unsupported aspect_ratio %q; use '16:9', '9:16', or '1:1'", ar)) + return ErrorResult(fmt.Sprintf("unsupported aspect_ratio %q; use '16:9' or '9:16'", ar)) } } @@ -186,24 +187,24 @@ func (t *CreateVideoTool) resolveConfig(ctx context.Context) (providerName, mode return providerName, model } -// callGeminiVideoGen uses the native Gemini generateContent API with VIDEO response modality. +// callGeminiVideoGen uses the Gemini predictLongRunning API for Veo video generation. +// Flow: POST predictLongRunning → poll operation → download video from URI. func (t *CreateVideoTool) callGeminiVideoGen(ctx context.Context, apiKey, apiBase, model, prompt string, duration int, aspectRatio string) ([]byte, *providers.Usage, error) { nativeBase := strings.TrimRight(apiBase, "/") nativeBase = strings.TrimSuffix(nativeBase, "/openai") - url := fmt.Sprintf("%s/models/%s:generateContent?key=%s", nativeBase, model, apiKey) - - genConfig := map[string]interface{}{ - "responseModalities": []string{"VIDEO"}, - "videoDuration": duration, - "aspectRatio": aspectRatio, - } + // 1. Start long-running video generation. + predictURL := fmt.Sprintf("%s/models/%s:predictLongRunning", nativeBase, model) body := map[string]interface{}{ - "contents": []map[string]interface{}{ - {"parts": []map[string]interface{}{{"text": prompt}}}, + "instances": []map[string]interface{}{ + {"prompt": prompt}, + }, + "parameters": map[string]interface{}{ + "aspectRatio": aspectRatio, + "durationSeconds": duration, + "personGeneration": "allow_all", }, - "generationConfig": genConfig, } jsonBody, err := json.Marshal(body) @@ -211,14 +212,14 @@ func (t *CreateVideoTool) callGeminiVideoGen(ctx context.Context, apiKey, apiBas return nil, nil, fmt.Errorf("marshal request: %w", err) } - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) + req, err := http.NewRequestWithContext(ctx, "POST", predictURL, bytes.NewReader(jsonBody)) if err != nil { return nil, nil, fmt.Errorf("create request: %w", err) } req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-goog-api-key", apiKey) - // Video generation can take a while. - client := &http.Client{Timeout: 300 * time.Second} + client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("http request: %w", err) @@ -233,49 +234,124 @@ func (t *CreateVideoTool) callGeminiVideoGen(ctx context.Context, apiKey, apiBas return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500)) } - // Parse Gemini response — look for inlineData with video MIME. - var gemResp struct { - Candidates []struct { - Content struct { - Parts []struct { - InlineData *struct { - MimeType string `json:"mimeType"` - Data string `json:"data"` - } `json:"inlineData"` - } `json:"parts"` - } `json:"content"` - } `json:"candidates"` - UsageMetadata *struct { - PromptTokenCount int `json:"promptTokenCount"` - CandidatesTokenCount int `json:"candidatesTokenCount"` - TotalTokenCount int `json:"totalTokenCount"` - } `json:"usageMetadata"` + var opResp struct { + Name string `json:"name"` + Done bool `json:"done"` } - if err := json.Unmarshal(respBody, &gemResp); err != nil { - return nil, nil, fmt.Errorf("parse response: %w", err) + if err := json.Unmarshal(respBody, &opResp); err != nil { + return nil, nil, fmt.Errorf("parse operation response: %w", err) + } + if opResp.Name == "" { + return nil, nil, fmt.Errorf("no operation name in response: %s", truncateBytes(respBody, 300)) } - for _, cand := range gemResp.Candidates { - for _, part := range cand.Content.Parts { - if part.InlineData != nil && strings.HasPrefix(part.InlineData.MimeType, "video/") { - videoBytes, err := base64.StdEncoding.DecodeString(part.InlineData.Data) - if err != nil { - return nil, nil, fmt.Errorf("decode base64: %w", err) - } - var usage *providers.Usage - if gemResp.UsageMetadata != nil { - usage = &providers.Usage{ - PromptTokens: gemResp.UsageMetadata.PromptTokenCount, - CompletionTokens: gemResp.UsageMetadata.CandidatesTokenCount, - TotalTokens: gemResp.UsageMetadata.TotalTokenCount, - } - } - return videoBytes, usage, nil - } + slog.Info("create_video: operation started", "operation", opResp.Name) + + // 2. Poll operation until done (max ~6 minutes, poll every 10s). + pollURL := fmt.Sprintf("%s/%s", nativeBase, opResp.Name) + const maxPolls = 40 + const pollInterval = 10 * time.Second + + var doneBody []byte + for i := 0; i < maxPolls; i++ { + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-time.After(pollInterval): } + + pollReq, err := http.NewRequestWithContext(ctx, "GET", pollURL, nil) + if err != nil { + return nil, nil, fmt.Errorf("create poll request: %w", err) + } + pollReq.Header.Set("x-goog-api-key", apiKey) + + pollResp, err := client.Do(pollReq) + if err != nil { + slog.Warn("create_video: poll error, retrying", "error", err, "attempt", i+1) + continue + } + + pollBody, _ := io.ReadAll(pollResp.Body) + pollResp.Body.Close() + + if pollResp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("poll API error %d: %s", pollResp.StatusCode, truncateBytes(pollBody, 500)) + } + + var pollOp struct { + Done bool `json:"done"` + Error json.RawMessage `json:"error"` + } + if err := json.Unmarshal(pollBody, &pollOp); err != nil { + return nil, nil, fmt.Errorf("parse poll response: %w", err) + } + + if pollOp.Error != nil && string(pollOp.Error) != "null" { + return nil, nil, fmt.Errorf("video generation error: %s", truncateBytes(pollOp.Error, 500)) + } + + if pollOp.Done { + doneBody = pollBody + break + } + + slog.Info("create_video: polling", "attempt", i+1, "done", pollOp.Done) } - return nil, nil, fmt.Errorf("no video data in Gemini response") + if doneBody == nil { + return nil, nil, fmt.Errorf("video generation timed out after %d polls", maxPolls) + } + + // 3. Extract video URI and download. + var result struct { + Response struct { + GenerateVideoResponse struct { + GeneratedSamples []struct { + Video struct { + URI string `json:"uri"` + MimeType string `json:"mimeType"` + } `json:"video"` + } `json:"generatedSamples"` + } `json:"generateVideoResponse"` + } `json:"response"` + } + if err := json.Unmarshal(doneBody, &result); err != nil { + return nil, nil, fmt.Errorf("parse final response: %w", err) + } + + samples := result.Response.GenerateVideoResponse.GeneratedSamples + if len(samples) == 0 || samples[0].Video.URI == "" { + return nil, nil, fmt.Errorf("no video in response: %s", truncateBytes(doneBody, 300)) + } + + videoURI := samples[0].Video.URI + slog.Info("create_video: downloading video", "uri", videoURI) + + dlReq, err := http.NewRequestWithContext(ctx, "GET", videoURI, nil) + if err != nil { + return nil, nil, fmt.Errorf("create download request: %w", err) + } + dlReq.Header.Set("x-goog-api-key", apiKey) + + dlClient := &http.Client{Timeout: 120 * time.Second} + dlResp, err := dlClient.Do(dlReq) + if err != nil { + return nil, nil, fmt.Errorf("download video: %w", err) + } + defer dlResp.Body.Close() + + if dlResp.StatusCode != http.StatusOK { + dlBody, _ := io.ReadAll(dlResp.Body) + return nil, nil, fmt.Errorf("download error %d: %s", dlResp.StatusCode, truncateBytes(dlBody, 300)) + } + + videoBytes, err := io.ReadAll(dlResp.Body) + if err != nil { + return nil, nil, fmt.Errorf("read video data: %w", err) + } + + return videoBytes, nil, nil } // callChatVideoGen tries OpenAI-compatible chat completions with video modality.