mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-13 12:14:40 +00:00
feat(ui): multi-skill upload with client-side validation (#149)
* feat(ui): multi-skill upload with client-side validation
Allow uploading multiple skill ZIP files at once with pre-upload
validation. JSZip parses each ZIP client-side to verify SKILL.md
presence, frontmatter format, and slug validity before upload.
- Add JSZip dependency (lazy-loaded, code-split ~30KB gzip)
- Create validate-skill-zip.ts mirroring server-side checks
- Rewrite skill-upload-dialog for multi-file with status badges
- Add concurrent validation, sequential upload with per-file progress
- Add empty SKILL.md check to backend upload handler
- Add i18n keys for all new UI strings (en/vi/zh)
* fix(ui): duplicate entries and validation hang in multi-skill upload
- Move pending list construction to assignment inside updater return
to prevent StrictMode double-invoke from pushing duplicates
- Wrap per-file validateSkillZip in try/catch so one failure doesn't
block Promise.all and leave entries stuck in "validating" state
* fix(ui): use static import for JSZip instead of dynamic import
Dynamic import("jszip") fails in browser - bare module specifiers
don't resolve at runtime. Use static import which Vite handles
via its module graph and code-splits automatically.
* feat(ui): add inline visibility toggle on skills table
Click the visibility badge on managed skills to cycle through
private → internal → public. File-based skills stay read-only.
* fix(ui): move dedup logic outside state updater in upload dialog
Avoids reading stale entries inside functional updater. Builds
pending list from current entries state before calling setEntries.
* fix(ui): auto-select first active agent when current agent unavailable
When agents load from API, if the current selected agent is not in the active agents list, automatically select the first available active agent instead of remaining unset. Prevents chat page from being unable to send messages when default agent selection is invalid.
* feat(ui): make agent display name editable in setup wizard
Allow users to customize the agent display name during onboarding instead of keeping it hardcoded to "GoClaw". Removed read-only state from the display name input and added a placeholder for guidance.
* feat: add document path enrichment and media filename support
Backend changes:
- enrichDocumentPaths() in agent/media.go: injects persisted file paths into <media:document> tags
- Document paths allow skills (e.g. pdf skill via exec) to access files directly
- chat.go: support new media format {path, filename} alongside legacy string paths
- Updated read_document tool description to guide agent on using path attribute
- Docker: add pypdf to Python dependencies for PDF processing
- Softened MUST language in read_* tool descriptions (changed to Call this)
Frontend changes:
- chat-input.tsx: attach filename with each uploaded file in media payload
- use-chat-send.ts: send media as {path, filename} objects instead of just paths
- i18n: add "uploaded_files" text in en, vi, zh locales
- chat-page.tsx: minor adjustment for media handling
Enables skills to process uploaded documents directly without intermediate copying.
This commit is contained in:
43 files changed
+7416
-145
No files matched your search
@@ -47,6 +47,7 @@ RUN set -eux; \
|
||||
fi; \
|
||||
if [ "$ENABLE_PYTHON" = "true" ]; then \
|
||||
apk add --no-cache python3 py3-pip; \
|
||||
pip3 install --break-system-packages pypdf; \
|
||||
fi
|
||||
|
||||
# Non-root user
|
||||
|
||||
@@ -14,6 +14,7 @@ services:
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
ENABLE_OTEL: "false"
|
||||
ENABLE_PYTHON: "true"
|
||||
ports:
|
||||
- "${GOCLAW_PORT:-18790}:18790"
|
||||
env_file:
|
||||
|
||||
@@ -247,6 +247,9 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
|
||||
}
|
||||
if len(docRefs) > 0 {
|
||||
ctx = tools.WithMediaDocRefs(ctx, docRefs)
|
||||
// Enrich the last user message with persisted file paths so skills can access
|
||||
// documents via exec (e.g. pypdf). Only for current-turn refs (just persisted).
|
||||
l.enrichDocumentPaths(messages, mediaRefs)
|
||||
}
|
||||
|
||||
// 2c. Collect audio MediaRefs (historical + current) for read_audio tool.
|
||||
|
||||
@@ -2,6 +2,7 @@ package agent
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -105,6 +106,63 @@ func (l *Loop) persistMedia(sessionKey string, files []bus.MediaFile) []provider
|
||||
return refs
|
||||
}
|
||||
|
||||
// enrichDocumentPaths updates the last user message to include persisted file paths
|
||||
// in <media:document> tags. This allows skills (e.g. pdf skill via exec) to access
|
||||
// the file directly, matching how Claude Code skills work with file paths.
|
||||
func (l *Loop) enrichDocumentPaths(messages []providers.Message, refs []providers.MediaRef) {
|
||||
if l.mediaStore == nil || len(messages) == 0 {
|
||||
return
|
||||
}
|
||||
// Find last user message
|
||||
lastIdx := -1
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
if messages[i].Role == "user" {
|
||||
lastIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if lastIdx < 0 {
|
||||
return
|
||||
}
|
||||
|
||||
content := messages[lastIdx].Content
|
||||
for _, ref := range refs {
|
||||
if ref.Kind != "document" {
|
||||
continue
|
||||
}
|
||||
p, err := l.mediaStore.LoadPath(ref.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Replace <media:document> or <media:document name="X"> with version that includes path.
|
||||
// The hint tells the agent the file is directly accessible (no copy needed).
|
||||
pathAttr := fmt.Sprintf(" path=%q", p)
|
||||
old1 := "<media:document>"
|
||||
new1 := "<media:document" + pathAttr + ">"
|
||||
if strings.Contains(content, old1) {
|
||||
content = strings.Replace(content, old1, new1, 1)
|
||||
continue
|
||||
}
|
||||
// For named variant, inject path attribute
|
||||
if idx := strings.Index(content, "<media:document name="); idx >= 0 {
|
||||
closeIdx := strings.Index(content[idx:], ">")
|
||||
if closeIdx >= 0 {
|
||||
tag := content[idx : idx+closeIdx]
|
||||
content = strings.Replace(content, tag+">", tag+pathAttr+">", 1)
|
||||
}
|
||||
}
|
||||
// For Slack variant with file= attribute
|
||||
if idx := strings.Index(content, "<media:document file="); idx >= 0 {
|
||||
closeIdx := strings.Index(content[idx:], ">")
|
||||
if closeIdx >= 0 {
|
||||
tag := content[idx : idx+closeIdx]
|
||||
content = strings.Replace(content, tag+">", tag+pathAttr+">", 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
messages[lastIdx].Content = content
|
||||
}
|
||||
|
||||
// mediaKindFromMime returns the media kind ("image", "video", "audio", "document")
|
||||
// based on MIME type prefix.
|
||||
func mediaKindFromMime(mime string) string {
|
||||
|
||||
@@ -73,11 +73,11 @@ var coreToolSummaries = map[string]string{
|
||||
"session_status": "Show session status (model, tokens, compaction count)",
|
||||
"sessions_history": "Fetch message history for a session",
|
||||
"sessions_send": "Send a message into another session",
|
||||
"read_image": "Analyze images attached to the conversation. MUST call this when you see <media:image> tags",
|
||||
"read_audio": "Analyze audio files attached to the conversation. MUST call this when you see <media:audio> tags",
|
||||
"read_video": "Analyze video files attached to the conversation. MUST call this when you see <media:video> tags",
|
||||
"read_image": "Analyze images attached to the conversation. Call this when you see <media:image> tags",
|
||||
"read_audio": "Analyze audio files attached to the conversation. Call this when you see <media:audio> tags",
|
||||
"read_video": "Analyze video files attached to the conversation. Call this when you see <media:video> tags",
|
||||
"create_video": "Generate videos from text descriptions using AI",
|
||||
"read_document": "Analyze documents (PDF, DOCX, etc.) attached to the conversation. MUST call this when you see <media:document> tags",
|
||||
"read_document": "Analyze documents (PDF, DOCX, etc.) attached to the conversation. Call this when you see <media:document> tags. If this tool fails, use a relevant skill instead (e.g. pdf skill with exec tool). The path attribute in <media:document path=\"...\"> is a directly accessible file in your workspace — use it directly, no need to copy",
|
||||
"create_image": "Generate images from text descriptions using AI",
|
||||
"create_audio": "Generate music or sound effects from text descriptions using AI",
|
||||
"knowledge_graph_search": "Search entities and traverse relationships in the knowledge graph",
|
||||
|
||||
@@ -36,12 +36,39 @@ func (m *ChatMethods) Register(router *gateway.MethodRouter) {
|
||||
router.Register(protocol.MethodChatInject, m.handleInject)
|
||||
}
|
||||
|
||||
// chatMediaItem represents a media file attached to a chat message.
|
||||
type chatMediaItem struct {
|
||||
Path string `json:"path"`
|
||||
Filename string `json:"filename,omitempty"`
|
||||
}
|
||||
|
||||
type chatSendParams struct {
|
||||
Message string `json:"message"`
|
||||
AgentID string `json:"agentId"`
|
||||
SessionKey string `json:"sessionKey"`
|
||||
Stream bool `json:"stream"`
|
||||
Media []string `json:"media,omitempty"` // local file paths from upload endpoint
|
||||
Message string `json:"message"`
|
||||
AgentID string `json:"agentId"`
|
||||
SessionKey string `json:"sessionKey"`
|
||||
Stream bool `json:"stream"`
|
||||
Media json.RawMessage `json:"media,omitempty"` // []string (legacy) or []chatMediaItem
|
||||
}
|
||||
|
||||
// parseMedia handles both legacy string paths and new {path,filename} objects.
|
||||
func (p *chatSendParams) parseMedia() []chatMediaItem {
|
||||
if len(p.Media) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Try new format: [{path, filename}]
|
||||
var items []chatMediaItem
|
||||
if err := json.Unmarshal(p.Media, &items); err == nil {
|
||||
return items
|
||||
}
|
||||
// Fallback: legacy ["path1", "path2"]
|
||||
var paths []string
|
||||
if err := json.Unmarshal(p.Media, &paths); err == nil {
|
||||
for _, path := range paths {
|
||||
items = append(items, chatMediaItem{Path: path})
|
||||
}
|
||||
return items
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ChatMethods) handleSend(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
|
||||
@@ -110,16 +137,20 @@ func (m *ChatMethods) handleSend(ctx context.Context, client *gateway.Client, re
|
||||
defer m.agents.UnregisterRun(runID)
|
||||
defer cancel()
|
||||
|
||||
// Convert string paths to bus.MediaFile with MIME detection.
|
||||
// Parse media items (supports both legacy string paths and new {path,filename} objects).
|
||||
items := params.parseMedia()
|
||||
|
||||
// Convert media items to bus.MediaFile with MIME detection.
|
||||
var mediaFiles []bus.MediaFile
|
||||
var mediaInfos []media.MediaInfo
|
||||
for _, p := range params.Media {
|
||||
mimeType := media.DetectMIMEType(p)
|
||||
mediaFiles = append(mediaFiles, bus.MediaFile{Path: p, MimeType: mimeType})
|
||||
for _, item := range items {
|
||||
mimeType := media.DetectMIMEType(item.Path)
|
||||
mediaFiles = append(mediaFiles, bus.MediaFile{Path: item.Path, MimeType: mimeType})
|
||||
mediaInfos = append(mediaInfos, media.MediaInfo{
|
||||
Type: media.MediaKindFromMime(mimeType),
|
||||
FilePath: p,
|
||||
FilePath: item.Path,
|
||||
ContentType: mimeType,
|
||||
FileName: item.Filename,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -89,6 +89,10 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, "failed to read SKILL.md")})
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(skillContent) == "" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, "SKILL.md is empty")})
|
||||
return
|
||||
}
|
||||
|
||||
name, description, slug, frontmatter := parseSkillFrontmatter(skillContent)
|
||||
if name == "" {
|
||||
|
||||
@@ -50,6 +50,7 @@ const documentMaxBytes = 20 * 1024 * 1024
|
||||
|
||||
// documentProviderPriority is the order in which providers are tried for document analysis.
|
||||
// Gemini has best native PDF support (50MB, 258 tokens/page).
|
||||
// "alibaba" is included as an alias for dashscope (common DB registration name).
|
||||
var documentProviderPriority = []string{"gemini", "anthropic", "openrouter", "dashscope"}
|
||||
|
||||
// documentModelDefaults maps provider names to preferred document-capable models.
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
# Google Gemini API Configuration
|
||||
|
||||
# ============================================================================
|
||||
# OPTION 1: Google AI Studio (Default - Recommended for most users)
|
||||
# ============================================================================
|
||||
# Get your API key: https://aistudio.google.com/apikey
|
||||
GEMINI_API_KEY=your_api_key_here
|
||||
|
||||
# ============================================================================
|
||||
# API Key Rotation (Optional - For high-volume usage)
|
||||
# ============================================================================
|
||||
# Add multiple API keys for automatic rotation on rate limit errors.
|
||||
# Free tier accounts are heavily rate-limited; rotation helps distribute load.
|
||||
#
|
||||
# Format: GEMINI_API_KEY_N where N is 2, 3, 4, etc.
|
||||
# The primary GEMINI_API_KEY is always used first.
|
||||
#
|
||||
# GEMINI_API_KEY_2=your_second_api_key
|
||||
# GEMINI_API_KEY_3=your_third_api_key
|
||||
# GEMINI_API_KEY_4=your_fourth_api_key
|
||||
#
|
||||
# Features:
|
||||
# - Auto-rotates on RESOURCE_EXHAUSTED / 429 errors
|
||||
# - 60-second cooldown per key after rate limit
|
||||
# - Logs rotation events with --verbose flag
|
||||
# - Backward compatible: single key still works
|
||||
|
||||
# ============================================================================
|
||||
# OPTION 2: Vertex AI (Google Cloud Platform)
|
||||
# ============================================================================
|
||||
# Uncomment these lines to use Vertex AI instead of Google AI Studio
|
||||
# GEMINI_USE_VERTEX=true
|
||||
# VERTEX_PROJECT_ID=your-gcp-project-id
|
||||
# VERTEX_LOCATION=us-central1
|
||||
|
||||
# ============================================================================
|
||||
# Model Selection (Optional)
|
||||
# ============================================================================
|
||||
# Override default models for specific capabilities
|
||||
# If not set, intelligent defaults are used based on task type
|
||||
|
||||
# --- Image Generation ---
|
||||
# Used by: --task generate (image)
|
||||
# Default: gemini-2.5-flash-image (Nano Banana Flash - fast, cost-effective)
|
||||
# Alternative: imagen-4.0-generate-001 (production quality)
|
||||
# NOTE: All image generation requires billing - no free tier available (limit: 0)
|
||||
# Options:
|
||||
# gemini-2.5-flash-image - Nano Banana Flash: fast, ~$1/1M tokens (DEFAULT)
|
||||
# gemini-3-pro-image-preview - Nano Banana Pro: 4K text, reasoning (requires billing)
|
||||
# imagen-4.0-generate-001 - Imagen 4 Standard: production quality (~$0.02/image)
|
||||
# imagen-4.0-ultra-generate-001 - Imagen 4 Ultra: maximum quality (~$0.04/image)
|
||||
# imagen-4.0-fast-generate-001 - Imagen 4 Fast: speed-optimized (~$0.01/image)
|
||||
# IMAGE_GEN_MODEL=gemini-2.5-flash-image
|
||||
|
||||
# --- Video Generation ---
|
||||
# Used by: --task generate-video (new capability)
|
||||
# Default: veo-3.1-generate-preview
|
||||
# NOTE: Video generation requires billing - no free tier fallback available
|
||||
# Options:
|
||||
# veo-3.1-generate-preview - Latest, native audio, frame control (requires billing)
|
||||
# veo-3.1-fast-generate-preview - Speed-optimized for business (requires billing)
|
||||
# veo-3.0-generate-001 - Stable, native audio, 8s videos (requires billing)
|
||||
# veo-3.0-fast-generate-001 - Stable fast variant (requires billing)
|
||||
# VIDEO_GEN_MODEL=veo-3.1-generate-preview
|
||||
|
||||
# --- Multimodal Analysis ---
|
||||
# Used by: --task analyze, transcribe, extract
|
||||
# Default: gemini-2.5-flash
|
||||
# Options:
|
||||
# gemini-3-pro-preview - Latest, agentic workflows, 1M context
|
||||
# gemini-2.5-flash - Best price/performance (recommended)
|
||||
# gemini-2.5-pro - Highest quality
|
||||
# MULTIMODAL_MODEL=gemini-2.5-flash
|
||||
|
||||
# --- Legacy Compatibility ---
|
||||
# Generic model override (use specific variables above instead)
|
||||
# GEMINI_MODEL=gemini-2.5-flash
|
||||
# GEMINI_IMAGE_GEN_MODEL=gemini-2.5-flash-image
|
||||
|
||||
# ============================================================================
|
||||
# Rate Limiting Configuration (Optional)
|
||||
# ============================================================================
|
||||
# Requests per minute limit (adjust based on your tier)
|
||||
# GEMINI_RPM_LIMIT=15
|
||||
|
||||
# Tokens per minute limit
|
||||
# GEMINI_TPM_LIMIT=4000000
|
||||
|
||||
# Requests per day limit
|
||||
# GEMINI_RPD_LIMIT=1500
|
||||
|
||||
# ============================================================================
|
||||
# Video Generation Options (Optional)
|
||||
# ============================================================================
|
||||
# Video duration in seconds (8s only for now)
|
||||
# VEO_DURATION=8
|
||||
|
||||
# Video resolution: 720p or 1080p
|
||||
# VEO_RESOLUTION=1080p
|
||||
|
||||
# Aspect ratio: 16:9, 9:16, 1:1 (16:9 is default)
|
||||
# VEO_ASPECT_RATIO=16:9
|
||||
|
||||
# Frame rate: 24fps (fixed for now)
|
||||
# VEO_FPS=24
|
||||
|
||||
# Enable native audio generation
|
||||
# VEO_AUDIO=true
|
||||
|
||||
# ============================================================================
|
||||
# Image Generation Options (Optional)
|
||||
# ============================================================================
|
||||
# Number of images to generate (1-4)
|
||||
# IMAGEN_NUM_IMAGES=1
|
||||
|
||||
# Image size: 1K or 2K (Ultra/Standard only)
|
||||
# IMAGEN_SIZE=1K
|
||||
|
||||
# Aspect ratio: 1:1, 16:9, 9:16, 4:3, 3:4
|
||||
# IMAGEN_ASPECT_RATIO=1:1
|
||||
|
||||
# Enable person generation (restricted in EEA, CH, UK)
|
||||
# IMAGEN_PERSON_GENERATION=true
|
||||
|
||||
# Add SynthID watermark (always enabled by default)
|
||||
# IMAGEN_WATERMARK=true
|
||||
|
||||
# ============================================================================
|
||||
# Processing Options (Optional)
|
||||
# ============================================================================
|
||||
# Video resolution mode: default or low-res
|
||||
# low-res uses ~100 tokens/second vs ~300 for default
|
||||
# GEMINI_VIDEO_RESOLUTION=default
|
||||
|
||||
# Audio quality: default (16 Kbps mono, auto-downsampled)
|
||||
# GEMINI_AUDIO_QUALITY=default
|
||||
|
||||
# PDF processing mode: inline (<20MB) or file-api (>20MB, automatic)
|
||||
# GEMINI_PDF_MODE=auto
|
||||
|
||||
# ============================================================================
|
||||
# Retry Configuration (Optional)
|
||||
# ============================================================================
|
||||
# Maximum retry attempts for failed requests
|
||||
# GEMINI_MAX_RETRIES=3
|
||||
|
||||
# Initial retry delay in seconds (uses exponential backoff)
|
||||
# GEMINI_RETRY_DELAY=1
|
||||
|
||||
# ============================================================================
|
||||
# Output Configuration (Optional)
|
||||
# ============================================================================
|
||||
# Default output directory for generated images
|
||||
# OUTPUT_DIR=./output
|
||||
|
||||
# Image output format (png or jpeg)
|
||||
# IMAGE_FORMAT=png
|
||||
|
||||
# Image quality for JPEG (1-100)
|
||||
# IMAGE_QUALITY=95
|
||||
|
||||
# ============================================================================
|
||||
# Context Caching (Optional)
|
||||
# ============================================================================
|
||||
# Enable context caching for repeated queries on same file
|
||||
# GEMINI_ENABLE_CACHING=true
|
||||
|
||||
# Cache TTL in seconds (default: 1800 = 30 minutes)
|
||||
# GEMINI_CACHE_TTL=1800
|
||||
|
||||
# ============================================================================
|
||||
# Logging (Optional)
|
||||
# ============================================================================
|
||||
# Log level: DEBUG, INFO, WARNING, ERROR, CRITICAL
|
||||
# LOG_LEVEL=INFO
|
||||
|
||||
# Log file path
|
||||
# LOG_FILE=./logs/gemini.log
|
||||
|
||||
# ============================================================================
|
||||
# Pricing Reference (as of 2025-11)
|
||||
# ============================================================================
|
||||
# Gemini 2.5 Flash: $1.00/1M input, $0.10/1M output
|
||||
# Gemini 2.5 Pro: $3.00/1M input, $12.00/1M output
|
||||
# Gemini 3 Pro: $2.00/1M input (<200k), $4.00 (>200k), $12/$18 output
|
||||
# Imagen 4: ~$0.01-$0.04 per image (varies by variant)
|
||||
# Veo 3: TBD (preview pricing)
|
||||
# Monitor: https://ai.google.dev/pricing
|
||||
|
||||
# ============================================================================
|
||||
# Notes
|
||||
# ============================================================================
|
||||
# 1. Never commit API keys to version control
|
||||
# 2. Add .env to .gitignore
|
||||
# 3. API keys can be restricted in Google Cloud Console
|
||||
# 4. Monitor usage at: https://aistudio.google.com/apikey
|
||||
# 5. Free tier limits: 15 RPM, 1M-4M TPM, 1,500 RPD
|
||||
# 6. Vertex AI requires GCP authentication via gcloud CLI
|
||||
# 7. Model defaults (Dec 2025):
|
||||
# - Image gen: gemini-2.5-flash-image (Nano Banana Flash - default)
|
||||
# - Image gen: imagen-4.0-generate-001 (alternative for production)
|
||||
# - Video gen: veo-3.1-generate-preview
|
||||
# - Analysis: gemini-2.5-flash
|
||||
# 8. Preview models (veo-3.1, gemini-3) may have API changes
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
name: ai-multimodal
|
||||
description: Analyze images/audio/video with Gemini API. Generate images (Imagen 4), videos (Veo 3). Use for vision analysis, transcription, OCR, design extraction, multimodal AI.
|
||||
---
|
||||
|
||||
# AI Multimodal
|
||||
|
||||
Process audio, images, videos, documents, and generate images/videos using Google Gemini's multimodal API.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
export GEMINI_API_KEY="your-key" # Get from https://aistudio.google.com/apikey
|
||||
pip install google-genai python-dotenv pillow
|
||||
```
|
||||
|
||||
### API Key Rotation (Optional)
|
||||
|
||||
For high-volume usage or when hitting rate limits, configure multiple API keys:
|
||||
|
||||
```bash
|
||||
# Primary key (required)
|
||||
export GEMINI_API_KEY="key1"
|
||||
|
||||
# Additional keys for rotation (optional)
|
||||
export GEMINI_API_KEY_2="key2"
|
||||
export GEMINI_API_KEY_3="key3"
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Auto-rotates on rate limit (429/RESOURCE_EXHAUSTED) errors
|
||||
- 60-second cooldown per key after rate limit
|
||||
- Logs rotation events with `--verbose` flag
|
||||
- Backward compatible: single key still works
|
||||
|
||||
## Quick Start
|
||||
|
||||
**Verify setup**: `python {baseDir}/scripts/check_setup.py`
|
||||
**Analyze media**: `python {baseDir}/scripts/gemini_batch_process.py --files <file> --task <analyze|transcribe|extract>`
|
||||
**Generate content**: `python {baseDir}/scripts/gemini_batch_process.py --task <generate|generate-video> --prompt "description"`
|
||||
|
||||
> **Stdin support**: You can pipe files directly via stdin (auto-detects PNG/JPG/PDF/WAV/MP3).
|
||||
> - `cat image.png | python {baseDir}/scripts/gemini_batch_process.py --task analyze --prompt "Describe this"`
|
||||
> - `python {baseDir}/scripts/gemini_batch_process.py --files image.png --task analyze` (traditional)
|
||||
|
||||
## Models
|
||||
|
||||
- **Image generation**: `imagen-4.0-generate-001` (standard), `imagen-4.0-ultra-generate-001` (quality), `imagen-4.0-fast-generate-001` (speed)
|
||||
- **Video generation**: `veo-3.1-generate-preview` (8s clips with audio)
|
||||
- **Analysis**: `gemini-2.5-flash` (recommended), `gemini-2.5-pro` (advanced)
|
||||
|
||||
## Scripts
|
||||
|
||||
- **`gemini_batch_process.py`**: CLI orchestrator for `transcribe|analyze|extract|generate|generate-video` that auto-resolves API keys, picks sensible default models per task, streams files inline vs File API, and saves structured outputs.
|
||||
- **`media_optimizer.py`**: ffmpeg/Pillow-based preflight tool that compresses/resizes/converts audio, image, and video inputs, enforces target sizes/bitrates, splits long clips into hour chunks.
|
||||
- **`document_converter.py`**: Gemini-powered converter that uploads PDFs/images/Office docs, applies a markdown-preserving prompt, batches multiple files.
|
||||
- **`check_setup.py`**: Interactive readiness checker that verifies Python deps and GEMINI_API_KEY availability.
|
||||
|
||||
Use `--help` for options.
|
||||
|
||||
## References
|
||||
|
||||
Load for detailed guidance:
|
||||
|
||||
| Topic | File | Description |
|
||||
|-------|------|-------------|
|
||||
| Music | `{baseDir}/references/music-generation.md` | Lyria RealTime API for background music generation. |
|
||||
| Audio | `{baseDir}/references/audio-processing.md` | Audio formats, transcription, non-speech analysis, TTS models. |
|
||||
| Images | `{baseDir}/references/vision-understanding.md` | Vision capabilities, captioning, OCR, multi-image workflows. |
|
||||
| Image Gen | `{baseDir}/references/image-generation.md` | Imagen 4, generate_images vs generate_content APIs, editing. |
|
||||
| Video | `{baseDir}/references/video-analysis.md` | Video analysis, clipping, FPS control, multi-video comparison. |
|
||||
| Video Gen | `{baseDir}/references/video-generation.md` | Veo models, text-to-video, image-to-video, camera control. |
|
||||
|
||||
## Limits
|
||||
|
||||
**Formats**: Audio (WAV/MP3/AAC, 9.5h), Images (PNG/JPEG/WEBP, 3.6k), Video (MP4/MOV, 6h), PDF (1k pages)
|
||||
**Size**: 20MB inline, 2GB File API
|
||||
**Important:**
|
||||
- Audio/video transcription >15 min: split into chunks (max 15 min each) and transcribe separately.
|
||||
- Video transcription: extract audio via ffmpeg first, then split and transcribe.
|
||||
|
||||
## Resources
|
||||
|
||||
- [API Docs](https://ai.google.dev/gemini-api/docs/)
|
||||
- [Pricing](https://ai.google.dev/pricing)
|
||||
@@ -0,0 +1,387 @@
|
||||
# Audio Processing Reference
|
||||
|
||||
Comprehensive guide for audio analysis and speech generation using Gemini API.
|
||||
|
||||
## Audio Understanding
|
||||
|
||||
### Supported Formats
|
||||
|
||||
| Format | MIME Type | Best Use |
|
||||
|--------|-----------|----------|
|
||||
| WAV | `audio/wav` | Uncompressed, highest quality |
|
||||
| MP3 | `audio/mp3` | Compressed, widely compatible |
|
||||
| AAC | `audio/aac` | Compressed, good quality |
|
||||
| FLAC | `audio/flac` | Lossless compression |
|
||||
| OGG Vorbis | `audio/ogg` | Open format |
|
||||
| AIFF | `audio/aiff` | Apple format |
|
||||
|
||||
### Specifications
|
||||
|
||||
- **Maximum length**: 9.5 hours per request
|
||||
- **Multiple files**: Unlimited count, combined max 9.5 hours
|
||||
- **Token rate**: 32 tokens/second (1 minute = 1,920 tokens)
|
||||
- **Processing**: Auto-downsampled to 16 Kbps mono
|
||||
- **File size limits**:
|
||||
- Inline: 20 MB max total request
|
||||
- File API: 2 GB per file, 20 GB project quota
|
||||
- Retention: 48 hours auto-delete
|
||||
- **Important:** if you are going to generate a transcript of the audio, and the audio length is longer than 15 minutes, the transcript often gets truncated due to output token limits in the Gemini API response. To get the full transcript, you need to split the audio into smaller chunks (max 15 minutes per chunk) and transcribe each segment for a complete transcript.
|
||||
|
||||
## Transcription
|
||||
|
||||
### Basic Transcription
|
||||
|
||||
```python
|
||||
from google import genai
|
||||
import os
|
||||
|
||||
client = genai.Client(api_key=os.getenv('GEMINI_API_KEY'))
|
||||
|
||||
# Upload audio
|
||||
myfile = client.files.upload(file='meeting.mp3')
|
||||
|
||||
# Transcribe
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Generate a transcript of the speech.', myfile]
|
||||
)
|
||||
print(response.text)
|
||||
```
|
||||
|
||||
### With Timestamps
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Generate transcript with timestamps in MM:SS format.', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
### Multi-Speaker Identification
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Transcribe with speaker labels. Format: [Speaker 1], [Speaker 2], etc.', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
### Segment-Specific Transcription
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Transcribe only the segment from 02:30 to 05:15.', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
## Audio Analysis
|
||||
|
||||
### Summarization
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Summarize key points in 5 bullets with timestamps.', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
### Non-Speech Audio Analysis
|
||||
|
||||
```python
|
||||
# Music analysis
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Identify the musical instruments and genre.', myfile]
|
||||
)
|
||||
|
||||
# Environmental sounds
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Identify all sounds: voices, music, ambient noise.', myfile]
|
||||
)
|
||||
|
||||
# Birdsong identification
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Identify bird species based on their calls.', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
### Timestamp-Based Analysis
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['What is discussed from 10:30 to 15:45? Provide key points.', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
## Input Methods
|
||||
|
||||
### File Upload (>20MB or Reuse)
|
||||
|
||||
```python
|
||||
# Upload once, use multiple times
|
||||
myfile = client.files.upload(file='large-audio.mp3')
|
||||
|
||||
# First query
|
||||
response1 = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Transcribe this', myfile]
|
||||
)
|
||||
|
||||
# Second query (reuses same file)
|
||||
response2 = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Summarize this', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
### Inline Data (<20MB)
|
||||
|
||||
```python
|
||||
from google.genai import types
|
||||
|
||||
with open('small-audio.mp3', 'rb') as f:
|
||||
audio_bytes = f.read()
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Describe this audio',
|
||||
types.Part.from_bytes(data=audio_bytes, mime_type='audio/mp3')
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Speech Generation (TTS)
|
||||
|
||||
### Available Models
|
||||
|
||||
| Model | Quality | Speed | Cost/1M tokens |
|
||||
|-------|---------|-------|----------------|
|
||||
| `gemini-2.5-flash-native-audio-preview-09-2025` | High | Fast | $10 |
|
||||
| `gemini-2.5-pro` TTS mode | Premium | Slower | $20 |
|
||||
|
||||
### Basic TTS
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-native-audio-preview-09-2025',
|
||||
contents='Generate audio: Welcome to today\'s episode.'
|
||||
)
|
||||
|
||||
# Save audio
|
||||
with open('output.wav', 'wb') as f:
|
||||
f.write(response.audio_data)
|
||||
```
|
||||
|
||||
### Controllable Voice Style
|
||||
|
||||
```python
|
||||
# Professional tone
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-native-audio-preview-09-2025',
|
||||
contents='Generate audio in a professional, clear tone: Welcome to our quarterly earnings call.'
|
||||
)
|
||||
|
||||
# Casual and friendly
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-native-audio-preview-09-2025',
|
||||
contents='Generate audio in a friendly, conversational tone: Hey there! Let\'s dive into today\'s topic.'
|
||||
)
|
||||
|
||||
# Narrative style
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-native-audio-preview-09-2025',
|
||||
contents='Generate audio in a narrative, storytelling tone: Once upon a time, in a land far away...'
|
||||
)
|
||||
```
|
||||
|
||||
### Voice Control Parameters
|
||||
|
||||
- **Style**: Professional, casual, narrative, conversational
|
||||
- **Pace**: Slow, normal, fast
|
||||
- **Tone**: Friendly, serious, enthusiastic
|
||||
- **Accent**: Natural language control (e.g., "British accent", "Southern drawl")
|
||||
|
||||
## Best Practices
|
||||
|
||||
### File Management
|
||||
|
||||
1. Use File API for files >20MB
|
||||
2. Use File API for repeated queries (saves tokens)
|
||||
3. Files auto-delete after 48 hours
|
||||
4. Clean up manually when done:
|
||||
```python
|
||||
client.files.delete(name=myfile.name)
|
||||
```
|
||||
|
||||
### Prompt Engineering
|
||||
|
||||
**Effective prompts**:
|
||||
- "Transcribe from 02:30 to 03:29 in MM:SS format"
|
||||
- "Identify speakers and extract dialogue with timestamps"
|
||||
- "Summarize key points with relevant timestamps"
|
||||
- "Transcribe and analyze sentiment for each speaker"
|
||||
|
||||
**Context improves accuracy**:
|
||||
- "This is a medical interview - use appropriate terminology"
|
||||
- "Transcribe this legal deposition with precise terminology"
|
||||
- "This is a technical podcast about machine learning"
|
||||
|
||||
**Combined tasks**:
|
||||
- "Transcribe and summarize in bullet points"
|
||||
- "Extract key quotes with timestamps and speaker labels"
|
||||
- "Transcribe and identify action items with timestamps"
|
||||
|
||||
### Cost Optimization
|
||||
|
||||
**Token calculation**:
|
||||
- 1 minute audio = 1,920 tokens
|
||||
- 1 hour audio = 115,200 tokens
|
||||
- 9.5 hours = 1,094,400 tokens
|
||||
|
||||
**Model selection**:
|
||||
- Use `gemini-2.5-flash` ($1/1M tokens) for most tasks
|
||||
- Upgrade to `gemini-2.5-pro` ($3/1M tokens) for complex analysis
|
||||
- For high-volume: `gemini-1.5-flash` ($0.70/1M tokens)
|
||||
|
||||
**Reduce costs**:
|
||||
- Process only relevant segments using timestamps
|
||||
- Use lower-quality audio when possible
|
||||
- Batch multiple short files in one request
|
||||
- Cache context for repeated queries
|
||||
|
||||
### Error Handling
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
def transcribe_with_retry(file_path, max_retries=3):
|
||||
"""Transcribe audio with exponential backoff retry"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
myfile = client.files.upload(file=file_path)
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Transcribe with timestamps', myfile]
|
||||
)
|
||||
return response.text
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
wait_time = 2 ** attempt
|
||||
print(f"Retry {attempt + 1} after {wait_time}s")
|
||||
time.sleep(wait_time)
|
||||
```
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### 1. Meeting Transcription
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Transcribe this meeting with:
|
||||
1. Speaker labels
|
||||
2. Timestamps for topic changes
|
||||
3. Action items highlighted
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 2. Podcast Summary
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Create podcast summary with:
|
||||
1. Main topics with timestamps
|
||||
2. Key quotes from each speaker
|
||||
3. Recommended episode highlights
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 3. Interview Analysis
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Analyze interview:
|
||||
1. Questions asked with timestamps
|
||||
2. Key responses from interviewee
|
||||
3. Overall sentiment and tone
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 4. Content Verification
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Verify audio content:
|
||||
1. Check for specific keywords or phrases
|
||||
2. Identify any compliance issues
|
||||
3. Note any concerning statements with timestamps
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 5. Multilingual Transcription
|
||||
|
||||
```python
|
||||
# Gemini auto-detects language
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Transcribe this audio and translate to English if needed.', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
## Token Costs
|
||||
|
||||
**Audio Input** (32 tokens/second):
|
||||
- 1 minute = 1,920 tokens
|
||||
- 10 minutes = 19,200 tokens
|
||||
- 1 hour = 115,200 tokens
|
||||
- 9.5 hours = 1,094,400 tokens
|
||||
|
||||
**Example costs** (Gemini 2.5 Flash at $1/1M):
|
||||
- 1 hour audio: 115,200 tokens = $0.12
|
||||
- Full day podcast (8 hours): 921,600 tokens = $0.92
|
||||
|
||||
## Limitations
|
||||
|
||||
- Maximum 9.5 hours per request
|
||||
- Auto-downsampled to 16 Kbps mono (quality loss)
|
||||
- Files expire after 48 hours
|
||||
- No real-time streaming support
|
||||
- Non-speech audio less accurate than speech
|
||||
|
||||
---
|
||||
|
||||
## Related References
|
||||
|
||||
**Current**: Audio Processing
|
||||
|
||||
**Related Capabilities**:
|
||||
- [Video Analysis](./video-analysis.md) - Extract audio from videos
|
||||
- [Video Generation](./video-generation.md) - Generate videos with native audio
|
||||
- [Image Understanding](./vision-understanding.md) - Analyze audio with visual context
|
||||
|
||||
**Back to**: [AI Multimodal Skill](../SKILL.md)
|
||||
@@ -0,0 +1,939 @@
|
||||
# Image Generation Reference
|
||||
|
||||
Comprehensive guide for image creation, editing, and composition using Imagen 4 and Gemini models ("Nano Banana").
|
||||
|
||||
> **Nano Banana** = Google's internal name for native image generation in Gemini API. Two variants: Nano Banana (Flash - speed) and Nano Banana Pro (3 Pro - quality with reasoning).
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
- **Text-to-Image**: Generate images from text prompts
|
||||
- **Image Editing**: Modify existing images with text instructions
|
||||
- **Multi-Image Composition**: Combine up to 14 reference images (Pro model)
|
||||
- **Iterative Refinement**: Multi-turn conversational refinement
|
||||
- **Aspect Ratios**: 10 formats (1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9, 21:9)
|
||||
- **Image Sizes**: 1K, 2K, 4K (uppercase K required)
|
||||
- **Quality Variants**: Standard/Ultra/Fast for different needs
|
||||
- **Text in Images**: Up to 25 chars optimal (4K text in Pro)
|
||||
- **Search Grounding**: Real-time data integration (Pro only)
|
||||
- **Thinking Mode**: Advanced reasoning for complex prompts (Pro only)
|
||||
|
||||
## Models
|
||||
|
||||
### Nano Banana (Default - Recommended)
|
||||
|
||||
**gemini-2.5-flash-image** - Nano Banana Flash ⭐ DEFAULT
|
||||
- Best for: Speed, high-volume generation, rapid prototyping
|
||||
- Quality: High
|
||||
- Context: 65,536 input / 32,768 output tokens
|
||||
- Speed: Fast (~5-10s per image)
|
||||
- Cost: ~$1/1M input tokens
|
||||
- Aspect Ratios: All 10 supported
|
||||
- Image Sizes: 1K, 2K, 4K
|
||||
- Status: Stable (Oct 2025)
|
||||
|
||||
**gemini-3-pro-image-preview** - Nano Banana Pro
|
||||
- Best for: Professional assets, 4K text rendering, complex prompts
|
||||
- Quality: Ultra (with advanced reasoning)
|
||||
- Context: 65,536 input / 32,768 output tokens
|
||||
- Speed: Medium
|
||||
- Cost: ~$2/1M text input, $0.134/image (resolution-dependent)
|
||||
- Multi-Image: Up to 14 reference images (6 objects + 5 humans)
|
||||
- Features: Thinking mode, Google Search grounding
|
||||
- Status: Preview (Nov 2025)
|
||||
|
||||
### Imagen 4 (Alternative - Production)
|
||||
|
||||
**imagen-4.0-generate-001** - Standard quality, balanced performance
|
||||
- Best for: Production workflows, marketing assets
|
||||
- Quality: High
|
||||
- Speed: Medium (~5-10s per image)
|
||||
- Cost: ~$0.02/image (estimated)
|
||||
- Output: 1-4 images per request
|
||||
- Resolution: 1K or 2K
|
||||
- Updated: June 2025
|
||||
|
||||
**imagen-4.0-ultra-generate-001** - Maximum quality
|
||||
- Best for: Final production, marketing assets, detailed artwork
|
||||
- Quality: Ultra (highest available)
|
||||
- Speed: Slow (~15-25s per image)
|
||||
- Cost: ~$0.04/image (estimated)
|
||||
- Output: 1-4 images per request
|
||||
- Resolution: 2K preferred
|
||||
- Updated: June 2025
|
||||
|
||||
**imagen-4.0-fast-generate-001** - Fastest generation
|
||||
- Best for: Rapid iteration, bulk generation, real-time use
|
||||
- Quality: Good
|
||||
- Speed: Fast (~2-5s per image)
|
||||
- Cost: ~$0.01/image (estimated)
|
||||
- Output: 1-4 images per request
|
||||
- Resolution: 1K
|
||||
- Updated: June 2025
|
||||
|
||||
### Legacy Models
|
||||
|
||||
**gemini-2.0-flash-preview-image-generation** - Legacy
|
||||
- Status: Deprecated (use Nano Banana or Imagen 4 instead)
|
||||
- Context: 32,768 input / 8,192 output tokens
|
||||
|
||||
## Model Comparison
|
||||
|
||||
| Model | Quality | Speed | Cost | Best For |
|
||||
|-------|---------|-------|------|----------|
|
||||
| gemini-2.5-flash-image | ⭐⭐⭐⭐ | 🚀 Fast | 💵 Low | **DEFAULT** - General use |
|
||||
| gemini-3-pro-image | ⭐⭐⭐⭐⭐ | 💡 Medium | 💰 Medium | Text/reasoning |
|
||||
| imagen-4.0-generate | ⭐⭐⭐⭐ | 💡 Medium | 💰 Medium | Production (alternative) |
|
||||
| imagen-4.0-ultra | ⭐⭐⭐⭐⭐ | 🐢 Slow | 💰💰 High | Marketing assets |
|
||||
| imagen-4.0-fast | ⭐⭐⭐ | 🚀 Fast | 💵 Low | Bulk generation |
|
||||
|
||||
**Selection Guide**:
|
||||
- **Default/General**: Use `gemini-2.5-flash-image` (fast, cost-effective)
|
||||
- **Production Quality**: Use `imagen-4.0-generate-001` (alternative for final assets)
|
||||
- **Marketing/Ultra Quality**: Use `imagen-4.0-ultra` for maximum quality
|
||||
- **Text-Heavy Images**: Use `gemini-3-pro-image-preview` for 4K text rendering
|
||||
- **Complex Prompts with Reasoning**: Use `gemini-3-pro-image-preview` with Thinking mode
|
||||
- **Real-time Data Integration**: Use `gemini-3-pro-image-preview` with Search grounding
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Basic Generation (Default - Nano Banana Flash)
|
||||
|
||||
```python
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
import os
|
||||
|
||||
client = genai.Client(api_key=os.getenv('GEMINI_API_KEY'))
|
||||
|
||||
# Nano Banana Flash - DEFAULT (fast, cost-effective)
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='A serene mountain landscape at sunset with snow-capped peaks',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['IMAGE'], # Uppercase required
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='16:9',
|
||||
image_size='2K' # 1K, 2K, 4K - uppercase K required
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Save images
|
||||
for i, part in enumerate(response.candidates[0].content.parts):
|
||||
if part.inline_data:
|
||||
with open(f'output-{i}.png', 'wb') as f:
|
||||
f.write(part.inline_data.data)
|
||||
```
|
||||
|
||||
### Alternative - Imagen 4 (Production Quality)
|
||||
|
||||
```python
|
||||
# Imagen 4 Standard - alternative for production workflows
|
||||
response = client.models.generate_images(
|
||||
model='imagen-4.0-generate-001',
|
||||
prompt='Professional product photography of smartphone',
|
||||
config=types.GenerateImagesConfig(
|
||||
numberOfImages=1,
|
||||
aspectRatio='16:9',
|
||||
imageSize='1K'
|
||||
)
|
||||
)
|
||||
|
||||
# Save Imagen 4 output
|
||||
for i, generated_image in enumerate(response.generated_images):
|
||||
with open(f'output-{i}.png', 'wb') as f:
|
||||
f.write(generated_image.image.image_bytes)
|
||||
```
|
||||
|
||||
### Imagen 4 Quality Variants
|
||||
|
||||
```python
|
||||
# Ultra quality (marketing assets)
|
||||
response = client.models.generate_images(
|
||||
model='imagen-4.0-ultra-generate-001',
|
||||
prompt='Professional product photography of smartphone',
|
||||
config=types.GenerateImagesConfig(
|
||||
numberOfImages=1,
|
||||
imageSize='2K' # Use 2K for ultra (Standard/Ultra only)
|
||||
)
|
||||
)
|
||||
|
||||
# Fast generation (bulk)
|
||||
# Note: Fast model doesn't support imageSize parameter
|
||||
response = client.models.generate_images(
|
||||
model='imagen-4.0-fast-generate-001',
|
||||
prompt='Quick concept sketch of robot character',
|
||||
config=types.GenerateImagesConfig(
|
||||
numberOfImages=4, # Generate multiple variants (default: 4)
|
||||
aspectRatio='1:1'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Nano Banana Pro (4K Text, Reasoning)
|
||||
|
||||
```python
|
||||
# Nano Banana Pro - for text rendering and complex prompts
|
||||
response = client.models.generate_content(
|
||||
model='gemini-3-pro-image-preview',
|
||||
contents='A futuristic cityscape with neon lights',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['IMAGE'], # Uppercase required
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='16:9',
|
||||
image_size='4K' # 4K text rendering
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Nano Banana Pro - with Thinking mode and Search grounding
|
||||
response = client.models.generate_content(
|
||||
model='gemini-3-pro-image-preview',
|
||||
contents='Current weather in Tokyo visualized as artistic infographic',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['TEXT', 'IMAGE'], # Both text and image
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='1:1',
|
||||
image_size='4K'
|
||||
)
|
||||
),
|
||||
tools=[{'google_search': {}}] # Enable search grounding
|
||||
)
|
||||
|
||||
# Save from content parts
|
||||
for i, part in enumerate(response.candidates[0].content.parts):
|
||||
if part.inline_data:
|
||||
with open(f'output-{i}.png', 'wb') as f:
|
||||
f.write(part.inline_data.data)
|
||||
```
|
||||
|
||||
### Multi-Image Reference (Nano Banana Pro)
|
||||
|
||||
```python
|
||||
from PIL import Image
|
||||
|
||||
# Up to 14 reference images (6 objects + 5 humans recommended)
|
||||
img1 = Image.open('style_ref.png')
|
||||
img2 = Image.open('color_ref.png')
|
||||
img3 = Image.open('composition_ref.png')
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-3-pro-image-preview',
|
||||
contents=[
|
||||
'Blend these reference styles into a cohesive hero image for a tech product',
|
||||
img1, img2, img3
|
||||
],
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['IMAGE'],
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='16:9',
|
||||
image_size='4K'
|
||||
)
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Multi-Turn Refinement Chat
|
||||
|
||||
```python
|
||||
# Conversational image refinement
|
||||
chat = client.chats.create(
|
||||
model='gemini-2.5-flash-image',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['TEXT', 'IMAGE']
|
||||
)
|
||||
)
|
||||
|
||||
# Initial generation
|
||||
response1 = chat.send_message('Create a minimalist logo for a coffee brand called "Brew"')
|
||||
|
||||
# Iterative refinement
|
||||
response2 = chat.send_message('Make the text bolder and add steam rising from the cup')
|
||||
response3 = chat.send_message('Change the color palette to warm earth tones')
|
||||
```
|
||||
|
||||
## API Differences
|
||||
|
||||
### Imagen 4 vs Nano Banana (Gemini Native)
|
||||
|
||||
| Feature | Imagen 4 | Nano Banana (Gemini) |
|
||||
|---------|----------|---------------------|
|
||||
| Method | `generate_images()` | `generate_content()` |
|
||||
| Config | `GenerateImagesConfig` | `GenerateContentConfig` |
|
||||
| Prompt param | `prompt` (string) | `contents` (string/list) |
|
||||
| Image count | `numberOfImages` (camelCase) | N/A (single per request) |
|
||||
| Aspect ratio | `aspectRatio` (camelCase) | `aspect_ratio` (snake_case) |
|
||||
| Size | `imageSize` | `image_size` |
|
||||
| Response | `generated_images[i].image.image_bytes` | `candidates[0].content.parts[i].inline_data.data` |
|
||||
| Multi-image input | ❌ | ✅ Up to 14 references |
|
||||
| Multi-turn chat | ❌ | ✅ Conversational |
|
||||
| Search grounding | ❌ | ✅ (Pro only) |
|
||||
| Thinking mode | ❌ | ✅ (Pro only) |
|
||||
| Text rendering | Limited | 4K (Pro) |
|
||||
|
||||
**Imagen 4** uses `generate_images()`:
|
||||
```python
|
||||
response = client.models.generate_images(
|
||||
model='imagen-4.0-generate-001',
|
||||
prompt='...',
|
||||
config=types.GenerateImagesConfig(
|
||||
numberOfImages=1, # camelCase
|
||||
aspectRatio='16:9', # camelCase
|
||||
imageSize='1K' # Standard/Ultra only
|
||||
)
|
||||
)
|
||||
# Access: response.generated_images[0].image.image_bytes
|
||||
```
|
||||
|
||||
**Nano Banana** uses `generate_content()`:
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image', # or gemini-3-pro-image-preview
|
||||
contents='...',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['IMAGE'], # Uppercase required
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='16:9', # snake_case
|
||||
image_size='2K' # 1K, 2K, 4K - uppercase K
|
||||
)
|
||||
)
|
||||
)
|
||||
# Access: response.candidates[0].content.parts[0].inline_data.data
|
||||
```
|
||||
|
||||
**Critical Notes**:
|
||||
1. `response_modalities` values MUST be uppercase: `'IMAGE'`, `'TEXT'`
|
||||
2. `image_size` value MUST have uppercase K: `'1K'`, `'2K'`, `'4K'`
|
||||
3. Imagen 4 Fast model doesn't support `imageSize` parameter
|
||||
|
||||
## Aspect Ratios
|
||||
|
||||
| Ratio | Resolution (1K) | Use Case | Token Cost |
|
||||
|-------|----------------|----------|------------|
|
||||
| 1:1 | 1024×1024 | Social media, avatars, icons | 1290 |
|
||||
| 2:3 | 682×1024 | Vertical portraits | 1290 |
|
||||
| 3:2 | 1024×682 | Horizontal portraits | 1290 |
|
||||
| 3:4 | 768×1024 | Vertical posters | 1290 |
|
||||
| 4:3 | 1024×768 | Traditional media | 1290 |
|
||||
| 4:5 | 819×1024 | Instagram portrait | 1290 |
|
||||
| 5:4 | 1024×819 | Horizontal photos | 1290 |
|
||||
| 9:16 | 576×1024 | Mobile/stories/reels | 1290 |
|
||||
| 16:9 | 1024×576 | Landscapes, banners, YouTube | 1290 |
|
||||
| 21:9 | 1024×438 | Ultrawide/cinematic | 1290 |
|
||||
|
||||
All ratios cost the same: 1,290 tokens per image (Gemini models).
|
||||
|
||||
## Response Modalities
|
||||
|
||||
### Image Only
|
||||
|
||||
```python
|
||||
config = types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='1:1'
|
||||
)
|
||||
```
|
||||
|
||||
### Text Only (No Image)
|
||||
|
||||
```python
|
||||
config = types.GenerateContentConfig(
|
||||
response_modalities=['text']
|
||||
)
|
||||
# Returns text description instead of generating image
|
||||
```
|
||||
|
||||
### Both Image and Text
|
||||
|
||||
```python
|
||||
config = types.GenerateContentConfig(
|
||||
response_modalities=['image', 'text'],
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
# Returns both generated image and description
|
||||
```
|
||||
|
||||
## Image Editing
|
||||
|
||||
### Modify Existing Image
|
||||
|
||||
```python
|
||||
import PIL.Image
|
||||
|
||||
# Load original
|
||||
img = PIL.Image.open('original.png')
|
||||
|
||||
# Edit with instructions
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=[
|
||||
'Add a red balloon floating in the sky',
|
||||
img
|
||||
],
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Style Transfer
|
||||
|
||||
```python
|
||||
img = PIL.Image.open('photo.jpg')
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=[
|
||||
'Transform this into an oil painting style',
|
||||
img
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Object Addition/Removal
|
||||
|
||||
```python
|
||||
# Add object
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=[
|
||||
'Add a vintage car parked on the street',
|
||||
img
|
||||
]
|
||||
)
|
||||
|
||||
# Remove object
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=[
|
||||
'Remove the person on the left side',
|
||||
img
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Multi-Image Composition
|
||||
|
||||
### Combine Multiple Images
|
||||
|
||||
```python
|
||||
img1 = PIL.Image.open('background.png')
|
||||
img2 = PIL.Image.open('foreground.png')
|
||||
img3 = PIL.Image.open('overlay.png')
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=[
|
||||
'Combine these images into a cohesive scene',
|
||||
img1,
|
||||
img2,
|
||||
img3
|
||||
],
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
**Note**: Recommended maximum 3 input images for best results.
|
||||
|
||||
## Prompt Engineering
|
||||
|
||||
### Core Principle: Narrative > Keywords
|
||||
|
||||
> **Nano Banana prompting**: Write like you're briefing a photographer, not providing SEO keywords. Narrative paragraphs outperform keyword lists.
|
||||
|
||||
❌ **Bad**: "cat, 4k, masterpiece, trending, professional, ultra detailed, cinematic"
|
||||
✅ **Good**: "A fluffy orange tabby cat with green eyes lounging on a sun-drenched windowsill. Soft morning light creates a warm glow. Shot with a 50mm lens at f/1.8 for shallow depth of field. Natural lighting, documentary photography style."
|
||||
|
||||
### Effective Prompt Structure
|
||||
|
||||
**Three key elements**:
|
||||
1. **Subject**: What to generate (be specific)
|
||||
2. **Context**: Environmental setting (lighting, location, time)
|
||||
3. **Style**: Artistic treatment (photography, illustration, etc.)
|
||||
|
||||
### Quality Modifiers
|
||||
|
||||
**Technical terms**:
|
||||
- "4K", "8K", "high resolution"
|
||||
- "HDR", "high dynamic range"
|
||||
- "professional photography"
|
||||
- "studio lighting"
|
||||
- "ultra detailed"
|
||||
|
||||
**Camera settings**:
|
||||
- "35mm lens", "50mm lens"
|
||||
- "shallow depth of field"
|
||||
- "wide angle shot"
|
||||
- "macro photography"
|
||||
- "golden hour lighting"
|
||||
|
||||
### Style Keywords
|
||||
|
||||
**Art styles**:
|
||||
- "oil painting", "watercolor", "sketch"
|
||||
- "digital art", "concept art"
|
||||
- "photorealistic", "hyperrealistic"
|
||||
- "minimalist", "abstract"
|
||||
- "cyberpunk", "steampunk", "fantasy"
|
||||
|
||||
**Mood and atmosphere**:
|
||||
- "dramatic lighting", "soft lighting"
|
||||
- "moody", "bright and cheerful"
|
||||
- "mysterious", "whimsical"
|
||||
- "dark and gritty", "pastel colors"
|
||||
|
||||
### Subject Description
|
||||
|
||||
**Be specific**:
|
||||
- ❌ "A cat"
|
||||
- ✅ "A fluffy orange tabby cat with green eyes"
|
||||
|
||||
**Add context**:
|
||||
- ❌ "A building"
|
||||
- ✅ "A modern glass skyscraper reflecting sunset clouds"
|
||||
|
||||
**Include details**:
|
||||
- ❌ "A person"
|
||||
- ✅ "A young woman in a red dress holding an umbrella"
|
||||
|
||||
### Composition and Framing
|
||||
|
||||
**Camera angles**:
|
||||
- "bird's eye view", "aerial shot"
|
||||
- "low angle", "high angle"
|
||||
- "close-up", "wide shot"
|
||||
- "centered composition"
|
||||
- "rule of thirds"
|
||||
|
||||
**Perspective**:
|
||||
- "first person view"
|
||||
- "third person perspective"
|
||||
- "isometric view"
|
||||
- "forced perspective"
|
||||
|
||||
### Text in Images
|
||||
|
||||
**Limitations**:
|
||||
- Maximum 25 characters total for optimal results
|
||||
- Up to 3 distinct text phrases
|
||||
- For 4K text rendering, use `gemini-3-pro-image-preview`
|
||||
|
||||
**Text prompt template**:
|
||||
```
|
||||
Image with text "[EXACT TEXT]" in [font style].
|
||||
Font: [style description].
|
||||
Color: [hex code like #FF5733].
|
||||
Position: [top/center/bottom].
|
||||
Background: [description].
|
||||
Context: [poster/sign/label].
|
||||
```
|
||||
|
||||
**Example**:
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-3-pro-image-preview', # Use Pro for better text
|
||||
contents='''
|
||||
Create a vintage travel poster with text "EXPLORE TOKYO" at the top.
|
||||
Font: Bold retro sans-serif, slightly condensed.
|
||||
Color: #F5E6D3 (cream white).
|
||||
Position: Top third of image.
|
||||
Background: Stylized Tokyo skyline with Mt. Fuji, sunset colors.
|
||||
Style: 1950s travel poster aesthetic, muted warm colors.
|
||||
'''
|
||||
)
|
||||
```
|
||||
|
||||
**Font keywords**:
|
||||
- "bold sans-serif", "handwritten script", "vintage letterpress"
|
||||
- "modern minimalist", "art deco", "neon sign"
|
||||
|
||||
### Nano Banana Prompt Techniques
|
||||
|
||||
| Technique | Example | Purpose |
|
||||
|-----------|---------|---------|
|
||||
| ALL CAPS emphasis | `The logo MUST be centered` | Force attention to critical requirements |
|
||||
| Hex colors | `#9F2B68` instead of "dark magenta" | Exact color control |
|
||||
| Negative constraints | `NEVER include text/watermarks. DO NOT add labels.` | Explicit exclusions |
|
||||
| Realism trigger | `Natural lighting, DOF. Captured with Canon EOS 90D DSLR.` | Photography authenticity |
|
||||
| Structured edits | `Make ALL edits: - [1] - [2] - [3]` | Multi-step changes |
|
||||
| Complex logic | `Kittens MUST have heterochromatic eyes matching fur colors` | Precise conditions |
|
||||
|
||||
**Prompt Templates**:
|
||||
|
||||
**Photorealistic**:
|
||||
```
|
||||
A [subject] in [location], [lens] lens. [Lighting] creates [mood]. [Details].
|
||||
[Camera angle]. Professional photography, natural lighting.
|
||||
```
|
||||
|
||||
**Illustration**:
|
||||
```
|
||||
[Art style] illustration of [subject]. [Color palette]. [Line style].
|
||||
[Background]. [Mood].
|
||||
```
|
||||
|
||||
**Product**:
|
||||
```
|
||||
[Product] on [surface]. Materials: [finish]. Lighting: [setup].
|
||||
Camera: [angle]. Background: [type]. Style: [commercial/lifestyle].
|
||||
```
|
||||
|
||||
## Advanced Techniques
|
||||
|
||||
### Iterative Refinement
|
||||
|
||||
```python
|
||||
# Initial generation
|
||||
response1 = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='A futuristic city skyline'
|
||||
)
|
||||
|
||||
# Save first version
|
||||
with open('v1.png', 'wb') as f:
|
||||
f.write(response1.candidates[0].content.parts[0].inline_data.data)
|
||||
|
||||
# Refine
|
||||
img = PIL.Image.open('v1.png')
|
||||
response2 = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=[
|
||||
'Add flying vehicles and neon signs',
|
||||
img
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Negative Prompts (Indirect)
|
||||
|
||||
```python
|
||||
# Instead of "no blur", be specific about what you want
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='A crystal clear, sharp photograph of a diamond ring with perfect focus and high detail'
|
||||
)
|
||||
```
|
||||
|
||||
### Consistent Style Across Images
|
||||
|
||||
```python
|
||||
base_prompt = "Digital art, vibrant colors, cel-shaded style, clean lines"
|
||||
|
||||
prompts = [
|
||||
f"{base_prompt}, a warrior character",
|
||||
f"{base_prompt}, a mage character",
|
||||
f"{base_prompt}, a rogue character"
|
||||
]
|
||||
|
||||
for i, prompt in enumerate(prompts):
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=prompt
|
||||
)
|
||||
# Save each character
|
||||
```
|
||||
|
||||
## Safety Settings
|
||||
|
||||
### Configure Safety Filters
|
||||
|
||||
```python
|
||||
config = types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
safety_settings=[
|
||||
types.SafetySetting(
|
||||
category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
|
||||
threshold=types.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE
|
||||
),
|
||||
types.SafetySetting(
|
||||
category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
|
||||
threshold=types.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Available Categories
|
||||
|
||||
- `HARM_CATEGORY_HATE_SPEECH`
|
||||
- `HARM_CATEGORY_DANGEROUS_CONTENT`
|
||||
- `HARM_CATEGORY_HARASSMENT`
|
||||
- `HARM_CATEGORY_SEXUALLY_EXPLICIT`
|
||||
|
||||
### Thresholds
|
||||
|
||||
- `BLOCK_NONE`: No blocking
|
||||
- `BLOCK_LOW_AND_ABOVE`: Block low probability and above
|
||||
- `BLOCK_MEDIUM_AND_ABOVE`: Block medium and above (default)
|
||||
- `BLOCK_ONLY_HIGH`: Block only high probability
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### 1. Marketing Assets
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='''Professional product photography:
|
||||
- Sleek smartphone on minimalist white surface
|
||||
- Dramatic side lighting creating subtle shadows
|
||||
- Shallow depth of field, crisp focus
|
||||
- Clean, modern aesthetic
|
||||
- 4K quality
|
||||
''',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='4:3'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### 2. Concept Art
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='''Fantasy concept art:
|
||||
- Ancient floating islands connected by chains
|
||||
- Waterfalls cascading into clouds below
|
||||
- Magical crystals glowing on the islands
|
||||
- Epic scale, dramatic lighting
|
||||
- Detailed digital painting style
|
||||
''',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### 3. Social Media Graphics
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='''Instagram post design:
|
||||
- Pastel gradient background (pink to blue)
|
||||
- Motivational quote layout
|
||||
- Modern minimalist style
|
||||
- Clean typography
|
||||
- Mobile-friendly composition
|
||||
''',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='1:1'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### 4. Illustration
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='''Children's book illustration:
|
||||
- Friendly cartoon dragon reading a book
|
||||
- Bright, cheerful colors
|
||||
- Soft, rounded shapes
|
||||
- Whimsical forest background
|
||||
- Warm, inviting atmosphere
|
||||
''',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='4:3'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### 5. UI/UX Mockups
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents='''Modern mobile app interface:
|
||||
- Clean dashboard design
|
||||
- Card-based layout
|
||||
- Soft shadows and gradients
|
||||
- Contemporary color scheme (blue and white)
|
||||
- Professional fintech aesthetic
|
||||
''',
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='9:16'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Prompt Quality
|
||||
|
||||
1. **Be specific**: More detail = better results
|
||||
2. **Order matters**: Most important elements first
|
||||
3. **Use examples**: Reference known styles or artists
|
||||
4. **Avoid contradictions**: Don't ask for opposing styles
|
||||
5. **Test and iterate**: Refine prompts based on results
|
||||
|
||||
### File Management
|
||||
|
||||
```python
|
||||
# Save with descriptive names
|
||||
timestamp = int(time.time())
|
||||
filename = f'generated_{timestamp}_{aspect_ratio}.png'
|
||||
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(image_data)
|
||||
```
|
||||
|
||||
### Cost Optimization
|
||||
|
||||
**Token costs**:
|
||||
- 1 image: 1,290 tokens = $0.00129 (Flash Image at $1/1M)
|
||||
- 10 images: 12,900 tokens = $0.0129
|
||||
- 100 images: 129,000 tokens = $0.129
|
||||
|
||||
**Strategies**:
|
||||
- Generate fewer iterations
|
||||
- Use text modality first to validate concept
|
||||
- Batch similar requests
|
||||
- Cache prompts for consistent style
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Safety Filter Blocking
|
||||
|
||||
```python
|
||||
try:
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash-image',
|
||||
contents=prompt
|
||||
)
|
||||
except Exception as e:
|
||||
# Check block reason
|
||||
if hasattr(e, 'prompt_feedback'):
|
||||
print(f"Blocked: {e.prompt_feedback.block_reason}")
|
||||
# Modify prompt and retry
|
||||
```
|
||||
|
||||
### Token Limit Exceeded
|
||||
|
||||
```python
|
||||
# Keep prompts concise
|
||||
if len(prompt) > 1000:
|
||||
# Truncate or simplify
|
||||
prompt = prompt[:1000]
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
### Imagen 4 Constraints
|
||||
- **Language**: English prompts only
|
||||
- **Prompt length**: Maximum 480 tokens
|
||||
- **Output**: 1-4 images per request
|
||||
- **Watermark**: All images include SynthID watermark
|
||||
- **Fast model**: No `imageSize` parameter support (fixed resolution)
|
||||
- **Text rendering**: Limited to ~25 characters for optimal results
|
||||
- **Regional restrictions**: Child images restricted in EEA, CH, UK
|
||||
- **Cannot replicate**: Specific people or copyrighted characters
|
||||
|
||||
### Nano Banana (Gemini) Constraints
|
||||
- **Language**: English prompts primary support
|
||||
- **Context**: 32K token window
|
||||
- **Multi-image**: Standard models ~3-5 refs; Pro up to 14 refs
|
||||
- **Text rendering**: Standard limited; Pro supports 4K text
|
||||
- **Watermark**: All images include SynthID watermark
|
||||
- **Case sensitivity**: `response_modalities` must be uppercase (`'IMAGE'`, `'TEXT'`)
|
||||
- **Size format**: `image_size` must have uppercase K (`'1K'`, `'2K'`, `'4K'`)
|
||||
|
||||
### General Limitations
|
||||
- Maximum 14 input images for composition (Pro only)
|
||||
- No video or animation generation (use Veo for video)
|
||||
- No real-time generation
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### aspect_ratio Parameter Error
|
||||
|
||||
**Error**: `Extra inputs are not permitted [type=extra_forbidden, input_value='1:1', input_type=str]`
|
||||
|
||||
**Cause**: The `aspect_ratio` parameter must be nested inside an `image_config` object, not passed directly to `GenerateContentConfig`.
|
||||
|
||||
**Incorrect Usage**:
|
||||
```python
|
||||
# ❌ This will fail
|
||||
config = types.GenerateContentConfig(
|
||||
response_modalities=['image'],
|
||||
aspect_ratio='16:9' # Wrong - not a direct parameter
|
||||
)
|
||||
```
|
||||
|
||||
**Correct Usage**:
|
||||
```python
|
||||
# ✅ Correct implementation
|
||||
config = types.GenerateContentConfig(
|
||||
response_modalities=['Image'], # Note: Capital 'I'
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Response Modality Case Sensitivity
|
||||
|
||||
The `response_modalities` parameter expects uppercase values:
|
||||
- ✅ Correct: `['IMAGE']`, `['TEXT']`, `['IMAGE', 'TEXT']`
|
||||
- ❌ Wrong: `['image']`, `['text']`, `['Image']`
|
||||
|
||||
### Image Size Parameter Not Supported
|
||||
|
||||
**Error**: `400 INVALID_ARGUMENT`
|
||||
|
||||
**Cause**: The `image_size` parameter in `ImageConfig` is not supported by all Nano Banana models.
|
||||
|
||||
**Solution**: Don't pass `image_size` unless explicitly needed. The API uses sensible defaults.
|
||||
|
||||
```python
|
||||
# ✅ Works - no image_size
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['IMAGE'],
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='16:9' # Only aspect_ratio
|
||||
)
|
||||
)
|
||||
|
||||
# ⚠️ May fail - with image_size (model-dependent)
|
||||
config=types.GenerateContentConfig(
|
||||
response_modalities=['IMAGE'],
|
||||
image_config=types.ImageConfig(
|
||||
aspect_ratio='16:9',
|
||||
image_size='2K' # Not supported by all models
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Multi-Image Reference Issues
|
||||
|
||||
**Problem**: Poor composition with multiple reference images
|
||||
|
||||
**Solutions**:
|
||||
1. Limit to 3-5 reference images for standard models
|
||||
2. Use Pro model for up to 14 references
|
||||
3. Collage multiple style refs into single image
|
||||
4. Provide clear textual descriptions of how to blend styles
|
||||
|
||||
---
|
||||
|
||||
## Related References
|
||||
|
||||
**Current**: Image Generation
|
||||
|
||||
**Related Capabilities**:
|
||||
- [Image Understanding](./vision-understanding.md) - Analyzing and editing reference images
|
||||
- [Video Generation](./video-generation.md) - Creating animated video content
|
||||
- [Audio Processing](./audio-processing.md) - Text-to-speech for multimedia
|
||||
|
||||
**Back to**: [AI Multimodal Skill](../SKILL.md)
|
||||
@@ -0,0 +1,311 @@
|
||||
# Music Generation Reference
|
||||
|
||||
Real-time music generation using Lyria RealTime via WebSocket API.
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
- **Real-time streaming**: Bidirectional WebSocket for continuous generation
|
||||
- **Dynamic control**: Modify music in real-time during generation
|
||||
- **Style steering**: Genre, mood, instrumentation guidance
|
||||
- **Audio output**: 48kHz stereo 16-bit PCM
|
||||
|
||||
## Model
|
||||
|
||||
**Lyria RealTime** (Experimental)
|
||||
- WebSocket-based streaming
|
||||
- Real-time parameter adjustment
|
||||
- Instrumental only (no vocals)
|
||||
- Watermarked output
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
from google import genai
|
||||
import asyncio
|
||||
|
||||
client = genai.Client(api_key=os.getenv('GEMINI_API_KEY'))
|
||||
|
||||
async def generate_music():
|
||||
async with client.aio.live.music.connect() as session:
|
||||
# Set style prompts with weights (0.0-1.0)
|
||||
await session.set_weighted_prompts([
|
||||
{"prompt": "Upbeat corporate background music", "weight": 0.8},
|
||||
{"prompt": "Modern electronic elements", "weight": 0.5}
|
||||
])
|
||||
|
||||
# Configure generation parameters
|
||||
await session.set_music_generation_config(
|
||||
guidance=4.0, # Prompt adherence (0.0-6.0)
|
||||
bpm=120, # Tempo (60-200)
|
||||
density=0.6, # Note density (0.0-1.0)
|
||||
brightness=0.5 # Tonal quality (0.0-1.0)
|
||||
)
|
||||
|
||||
# Start playback and collect audio
|
||||
await session.play()
|
||||
|
||||
audio_chunks = []
|
||||
async for chunk in session:
|
||||
audio_chunks.append(chunk.audio_data)
|
||||
|
||||
return b''.join(audio_chunks)
|
||||
```
|
||||
|
||||
### JavaScript
|
||||
|
||||
```javascript
|
||||
const client = new GenaiClient({ apiKey: process.env.GEMINI_API_KEY });
|
||||
|
||||
async function generateMusic() {
|
||||
const session = await client.live.music.connect();
|
||||
|
||||
await session.setWeightedPrompts([
|
||||
{ prompt: "Calm ambient background", weight: 0.9 },
|
||||
{ prompt: "Nature sounds influence", weight: 0.3 }
|
||||
]);
|
||||
|
||||
await session.setMusicGenerationConfig({
|
||||
guidance: 3.5,
|
||||
bpm: 80,
|
||||
density: 0.4,
|
||||
brightness: 0.6
|
||||
});
|
||||
|
||||
session.onAudio((audioChunk) => {
|
||||
// Process 48kHz stereo PCM audio
|
||||
audioBuffer.push(audioChunk);
|
||||
});
|
||||
|
||||
await session.play();
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration Parameters
|
||||
|
||||
| Parameter | Range | Default | Description |
|
||||
|-----------|-------|---------|-------------|
|
||||
| `guidance` | 0.0-6.0 | 4.0 | Prompt adherence (higher = stricter) |
|
||||
| `bpm` | 60-200 | 120 | Tempo in beats per minute |
|
||||
| `density` | 0.0-1.0 | 0.5 | Note/sound density |
|
||||
| `brightness` | 0.0-1.0 | 0.5 | Tonal quality (higher = brighter) |
|
||||
| `scale` | 12 keys | C Major | Musical key |
|
||||
| `mute_bass` | bool | false | Remove bass elements |
|
||||
| `mute_drums` | bool | false | Remove drum elements |
|
||||
| `mode` | enum | QUALITY | QUALITY, DIVERSITY, VOCALIZATION |
|
||||
| `temperature` | 0.0-2.0 | 1.0 | Sampling randomness |
|
||||
| `top_k` | int | 40 | Sampling top-k |
|
||||
| `seed` | int | random | Reproducibility seed |
|
||||
|
||||
## Weighted Prompts
|
||||
|
||||
Control generation direction with weighted prompts:
|
||||
|
||||
```python
|
||||
await session.set_weighted_prompts([
|
||||
{"prompt": "Main style description", "weight": 1.0}, # Primary
|
||||
{"prompt": "Secondary influence", "weight": 0.5}, # Supporting
|
||||
{"prompt": "Subtle element", "weight": 0.2} # Accent
|
||||
])
|
||||
```
|
||||
|
||||
**Weight guidelines**:
|
||||
- 0.8-1.0: Dominant influence
|
||||
- 0.5-0.7: Secondary contribution
|
||||
- 0.2-0.4: Subtle accent
|
||||
- 0.0-0.1: Minimal effect
|
||||
|
||||
## Style Prompts by Use Case
|
||||
|
||||
### Corporate/Marketing
|
||||
|
||||
```python
|
||||
prompts = [
|
||||
{"prompt": "Professional corporate background music, modern", "weight": 0.9},
|
||||
{"prompt": "Uplifting, optimistic mood", "weight": 0.6},
|
||||
{"prompt": "Clean production, minimal complexity", "weight": 0.5}
|
||||
]
|
||||
config = {"bpm": 100, "brightness": 0.6, "density": 0.5}
|
||||
```
|
||||
|
||||
### Social Media/Short-form
|
||||
|
||||
```python
|
||||
prompts = [
|
||||
{"prompt": "Trending pop electronic beat", "weight": 0.9},
|
||||
{"prompt": "Energetic, catchy rhythm", "weight": 0.7},
|
||||
{"prompt": "Bass-heavy, punchy", "weight": 0.5}
|
||||
]
|
||||
config = {"bpm": 128, "brightness": 0.7, "density": 0.7}
|
||||
```
|
||||
|
||||
### Emotional/Cinematic
|
||||
|
||||
```python
|
||||
prompts = [
|
||||
{"prompt": "Cinematic orchestral underscore", "weight": 0.9},
|
||||
{"prompt": "Emotional, inspiring", "weight": 0.7},
|
||||
{"prompt": "Building tension and release", "weight": 0.5}
|
||||
]
|
||||
config = {"bpm": 70, "brightness": 0.4, "density": 0.4}
|
||||
```
|
||||
|
||||
### Ambient/Background
|
||||
|
||||
```python
|
||||
prompts = [
|
||||
{"prompt": "Calm ambient soundscape", "weight": 0.9},
|
||||
{"prompt": "Minimal, atmospheric", "weight": 0.6},
|
||||
{"prompt": "Lo-fi textures", "weight": 0.4}
|
||||
]
|
||||
config = {"bpm": 80, "brightness": 0.4, "density": 0.3}
|
||||
```
|
||||
|
||||
## Real-time Transitions
|
||||
|
||||
Smoothly transition between styles during generation:
|
||||
|
||||
```python
|
||||
async def dynamic_music_generation():
|
||||
async with client.aio.live.music.connect() as session:
|
||||
# Start with intro style
|
||||
await session.set_weighted_prompts([
|
||||
{"prompt": "Soft ambient intro", "weight": 0.9}
|
||||
])
|
||||
await session.play()
|
||||
|
||||
# Collect intro (4 seconds)
|
||||
intro_chunks = []
|
||||
for _ in range(192): # ~4 seconds at 48kHz
|
||||
chunk = await session.__anext__()
|
||||
intro_chunks.append(chunk.audio_data)
|
||||
|
||||
# Transition to main section
|
||||
await session.set_weighted_prompts([
|
||||
{"prompt": "Building energy", "weight": 0.7},
|
||||
{"prompt": "Full beat drop", "weight": 0.5}
|
||||
])
|
||||
|
||||
# Continue with new style...
|
||||
```
|
||||
|
||||
## Output Specifications
|
||||
|
||||
- **Format**: Raw 16-bit PCM
|
||||
- **Sample Rate**: 48,000 Hz
|
||||
- **Channels**: 2 (stereo)
|
||||
- **Bit Depth**: 16 bits
|
||||
- **Watermarking**: Always enabled (SynthID)
|
||||
|
||||
### Save to WAV
|
||||
|
||||
```python
|
||||
import wave
|
||||
|
||||
def save_pcm_to_wav(pcm_data, filename):
|
||||
with wave.open(filename, 'wb') as wav_file:
|
||||
wav_file.setnchannels(2) # Stereo
|
||||
wav_file.setsampwidth(2) # 16-bit
|
||||
wav_file.setframerate(48000) # 48kHz
|
||||
wav_file.writeframes(pcm_data)
|
||||
```
|
||||
|
||||
### Convert to MP3
|
||||
|
||||
```bash
|
||||
# Using FFmpeg
|
||||
ffmpeg -f s16le -ar 48000 -ac 2 -i input.pcm output.mp3
|
||||
```
|
||||
|
||||
## Integration with Video Production
|
||||
|
||||
### Generate Background Music for Video
|
||||
|
||||
```python
|
||||
async def generate_video_background(duration_seconds, mood):
|
||||
"""Generate background music matching video length"""
|
||||
|
||||
# Configure for video background
|
||||
prompts = [
|
||||
{"prompt": f"{mood} background music for video", "weight": 0.9},
|
||||
{"prompt": "Non-distracting, supportive underscore", "weight": 0.6}
|
||||
]
|
||||
|
||||
async with client.aio.live.music.connect() as session:
|
||||
await session.set_weighted_prompts(prompts)
|
||||
await session.set_music_generation_config(
|
||||
guidance=4.0,
|
||||
density=0.4, # Keep sparse for background
|
||||
brightness=0.5
|
||||
)
|
||||
await session.play()
|
||||
|
||||
# Calculate chunks needed (48kHz stereo = 192000 bytes/second)
|
||||
total_chunks = duration_seconds * 48000 // 512 # Chunk size estimate
|
||||
|
||||
audio_data = []
|
||||
async for i, chunk in enumerate(session):
|
||||
audio_data.append(chunk.audio_data)
|
||||
if i >= total_chunks:
|
||||
break
|
||||
|
||||
return b''.join(audio_data)
|
||||
```
|
||||
|
||||
### Sync with Storyboard Timing
|
||||
|
||||
```python
|
||||
async def generate_scene_music(scenes):
|
||||
"""Generate music with transitions matching scene changes"""
|
||||
|
||||
all_audio = []
|
||||
|
||||
async with client.aio.live.music.connect() as session:
|
||||
for scene in scenes:
|
||||
# Update style for each scene
|
||||
await session.set_weighted_prompts([
|
||||
{"prompt": scene['mood'], "weight": 0.9},
|
||||
{"prompt": scene['style'], "weight": 0.5}
|
||||
])
|
||||
|
||||
if scene['index'] == 0:
|
||||
await session.play()
|
||||
|
||||
# Collect audio for scene duration
|
||||
chunks = int(scene['duration'] * 48000 / 512)
|
||||
for _ in range(chunks):
|
||||
chunk = await session.__anext__()
|
||||
all_audio.append(chunk.audio_data)
|
||||
|
||||
return b''.join(all_audio)
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Instrumental only**: No vocal/singing generation
|
||||
- **WebSocket required**: Real-time streaming connection
|
||||
- **Safety filtering**: Prompts undergo safety review
|
||||
- **Watermarking**: All output contains SynthID watermark
|
||||
- **Experimental**: API may change
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Buffer audio**: Implement robust buffering for smooth playback
|
||||
2. **Gradual transitions**: Avoid drastic prompt changes mid-stream
|
||||
3. **Sparse for backgrounds**: Lower density for video backgrounds
|
||||
4. **Test prompts**: Iterate on prompt combinations
|
||||
5. **Cross-fade transitions**: Blend audio at style changes
|
||||
6. **Match video mood**: Align music tempo/energy with visuals
|
||||
|
||||
## Resources
|
||||
|
||||
- [Lyria RealTime Docs](https://ai.google.dev/gemini-api/docs/music-generation)
|
||||
- [Audio Processing Guide](./audio-processing.md)
|
||||
- [Video Generation](./video-generation.md)
|
||||
|
||||
---
|
||||
|
||||
**Related**: [Audio Processing](./audio-processing.md) | [Video Generation](./video-generation.md)
|
||||
|
||||
**Back to**: [AI Multimodal Skill](../SKILL.md)
|
||||
@@ -0,0 +1,515 @@
|
||||
# Video Analysis Reference
|
||||
|
||||
Comprehensive guide for video understanding, temporal analysis, and YouTube processing using Gemini API.
|
||||
|
||||
> **Note**: This guide covers video *analysis* (understanding existing videos). For video *generation* (creating new videos), see [Video Generation Reference](./video-generation.md).
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
- **Video Summarization**: Create concise summaries
|
||||
- **Question Answering**: Answer specific questions about content
|
||||
- **Transcription**: Audio transcription with visual descriptions
|
||||
- **Timestamp References**: Query specific moments (MM:SS format)
|
||||
- **Video Clipping**: Process specific segments
|
||||
- **Scene Detection**: Identify scene changes and transitions
|
||||
- **Multiple Videos**: Compare up to 10 videos (2.5+)
|
||||
- **YouTube Support**: Analyze YouTube videos directly
|
||||
- **Custom Frame Rate**: Adjust FPS sampling
|
||||
|
||||
## Supported Formats
|
||||
|
||||
- MP4, MPEG, MOV, AVI, FLV, MPG, WebM, WMV, 3GPP
|
||||
|
||||
## Model Selection
|
||||
|
||||
### Gemini 3 Series (Latest)
|
||||
- **gemini-3-pro-preview**: Latest, agentic workflows, 1M context, dynamic thinking
|
||||
|
||||
### Gemini 2.5 Series (Recommended)
|
||||
- **gemini-2.5-pro**: Best quality, 1M-2M context
|
||||
- **gemini-2.5-flash**: Balanced, 1M-2M context (recommended)
|
||||
|
||||
### Context Windows
|
||||
- **2M token models**: ~2 hours (default) or ~6 hours (low-res)
|
||||
- **1M token models**: ~1 hour (default) or ~3 hours (low-res)
|
||||
|
||||
## Basic Video Analysis
|
||||
|
||||
### Local Video
|
||||
|
||||
```python
|
||||
from google import genai
|
||||
import os
|
||||
|
||||
client = genai.Client(api_key=os.getenv('GEMINI_API_KEY'))
|
||||
|
||||
# Upload video (File API for >20MB)
|
||||
myfile = client.files.upload(file='video.mp4')
|
||||
|
||||
# Wait for processing
|
||||
import time
|
||||
while myfile.state.name == 'PROCESSING':
|
||||
time.sleep(1)
|
||||
myfile = client.files.get(name=myfile.name)
|
||||
|
||||
if myfile.state.name == 'FAILED':
|
||||
raise ValueError('Video processing failed')
|
||||
|
||||
# Analyze
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Summarize this video in 3 key points', myfile]
|
||||
)
|
||||
print(response.text)
|
||||
```
|
||||
|
||||
### YouTube Video
|
||||
|
||||
```python
|
||||
from google.genai import types
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Summarize the main topics discussed',
|
||||
types.Part.from_uri(
|
||||
uri='https://www.youtube.com/watch?v=VIDEO_ID',
|
||||
mime_type='video/mp4'
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Inline Video (<20MB)
|
||||
|
||||
```python
|
||||
with open('short-clip.mp4', 'rb') as f:
|
||||
video_bytes = f.read()
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'What happens in this video?',
|
||||
types.Part.from_bytes(data=video_bytes, mime_type='video/mp4')
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Video Clipping
|
||||
|
||||
```python
|
||||
# Analyze specific time range
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Summarize this segment',
|
||||
types.Part.from_video_metadata(
|
||||
file_uri=myfile.uri,
|
||||
start_offset='40s',
|
||||
end_offset='80s'
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Custom Frame Rate
|
||||
|
||||
```python
|
||||
# Lower FPS for static content (saves tokens)
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Analyze this presentation',
|
||||
types.Part.from_video_metadata(
|
||||
file_uri=myfile.uri,
|
||||
fps=0.5 # Sample every 2 seconds
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
# Higher FPS for fast-moving content
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Analyze rapid movements in this sports video',
|
||||
types.Part.from_video_metadata(
|
||||
file_uri=myfile.uri,
|
||||
fps=5 # Sample 5 times per second
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Multiple Videos (2.5+)
|
||||
|
||||
```python
|
||||
video1 = client.files.upload(file='demo1.mp4')
|
||||
video2 = client.files.upload(file='demo2.mp4')
|
||||
|
||||
# Wait for processing
|
||||
for video in [video1, video2]:
|
||||
while video.state.name == 'PROCESSING':
|
||||
time.sleep(1)
|
||||
video = client.files.get(name=video.name)
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-pro',
|
||||
contents=[
|
||||
'Compare these two product demos. Which explains features better?',
|
||||
video1,
|
||||
video2
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Temporal Understanding
|
||||
|
||||
### Timestamp-Based Questions
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'What happens at 01:15 and how does it relate to 02:30?',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Timeline Creation
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Create a timeline with timestamps:
|
||||
- Key events
|
||||
- Scene changes
|
||||
- Important moments
|
||||
Format: MM:SS - Description
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Scene Detection
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Identify all scene changes with timestamps and describe each scene',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Transcription
|
||||
|
||||
### Basic Transcription
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Transcribe the audio from this video',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### With Visual Descriptions
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Transcribe with visual context:
|
||||
- Audio transcription
|
||||
- Visual descriptions of important moments
|
||||
- Timestamps for salient events
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Speaker Identification
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Transcribe with speaker labels and timestamps',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### 1. Video Summarization
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Summarize this video:
|
||||
1. Main topic and purpose
|
||||
2. Key points with timestamps
|
||||
3. Conclusion or call-to-action
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 2. Educational Content
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Create educational materials:
|
||||
1. List key concepts taught
|
||||
2. Create 5 quiz questions with answers
|
||||
3. Provide timestamp for each concept
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 3. Action Detection
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'List all actions performed in this tutorial with timestamps',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 4. Content Moderation
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Review video content:
|
||||
1. Identify any problematic content
|
||||
2. Note timestamps of concerns
|
||||
3. Provide content rating recommendation
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 5. Interview Analysis
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Analyze interview:
|
||||
1. Questions asked (timestamps)
|
||||
2. Key responses
|
||||
3. Candidate body language and demeanor
|
||||
4. Overall assessment
|
||||
''',
|
||||
myfile
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 6. Sports Analysis
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Analyze sports video:
|
||||
1. Key plays with timestamps
|
||||
2. Player movements and positioning
|
||||
3. Game strategy observations
|
||||
''',
|
||||
types.Part.from_video_metadata(
|
||||
file_uri=myfile.uri,
|
||||
fps=5 # Higher FPS for fast action
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## YouTube Specific Features
|
||||
|
||||
### Public Video Requirements
|
||||
|
||||
- Video must be public (not private or unlisted)
|
||||
- No age-restricted content
|
||||
- Valid video ID required
|
||||
|
||||
### Usage Example
|
||||
|
||||
```python
|
||||
# YouTube URL
|
||||
youtube_uri = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Create chapter markers with timestamps',
|
||||
types.Part.from_uri(uri=youtube_uri, mime_type='video/mp4')
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Rate Limits
|
||||
|
||||
- **Free tier**: 8 hours of YouTube video per day
|
||||
- **Paid tier**: No length-based limits
|
||||
- Public videos only
|
||||
|
||||
## Token Calculation
|
||||
|
||||
Video tokens depend on resolution and FPS:
|
||||
|
||||
**Default resolution** (~300 tokens/second):
|
||||
- 1 minute = 18,000 tokens
|
||||
- 10 minutes = 180,000 tokens
|
||||
- 1 hour = 1,080,000 tokens
|
||||
|
||||
**Low resolution** (~100 tokens/second):
|
||||
- 1 minute = 6,000 tokens
|
||||
- 10 minutes = 60,000 tokens
|
||||
- 1 hour = 360,000 tokens
|
||||
|
||||
**Context windows**:
|
||||
- 2M tokens ≈ 2 hours (default) or 6 hours (low-res)
|
||||
- 1M tokens ≈ 1 hour (default) or 3 hours (low-res)
|
||||
|
||||
## Best Practices
|
||||
|
||||
### File Management
|
||||
|
||||
1. Use File API for videos >20MB (most videos)
|
||||
2. Wait for ACTIVE state before analysis
|
||||
3. Files auto-delete after 48 hours
|
||||
4. Clean up manually:
|
||||
```python
|
||||
client.files.delete(name=myfile.name)
|
||||
```
|
||||
|
||||
### Optimization Strategies
|
||||
|
||||
**Reduce token usage**:
|
||||
- Process specific segments using start/end offsets
|
||||
- Use lower FPS for static content
|
||||
- Use low-resolution mode for long videos
|
||||
- Split very long videos into chunks
|
||||
|
||||
**Improve accuracy**:
|
||||
- Provide context in prompts
|
||||
- Use higher FPS for fast-moving content
|
||||
- Use Pro model for complex analysis
|
||||
- Be specific about what to extract
|
||||
|
||||
### Prompt Engineering
|
||||
|
||||
**Effective prompts**:
|
||||
- "Summarize key points with timestamps in MM:SS format"
|
||||
- "Identify all scene changes and describe each scene"
|
||||
- "Extract action items mentioned with timestamps"
|
||||
- "Compare these two videos on: X, Y, Z criteria"
|
||||
|
||||
**Structured output**:
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
from typing import List
|
||||
|
||||
class VideoEvent(BaseModel):
|
||||
timestamp: str # MM:SS format
|
||||
description: str
|
||||
category: str
|
||||
|
||||
class VideoAnalysis(BaseModel):
|
||||
summary: str
|
||||
events: List[VideoEvent]
|
||||
duration: str
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Analyze this video', myfile],
|
||||
config=genai.types.GenerateContentConfig(
|
||||
response_mime_type='application/json',
|
||||
response_schema=VideoAnalysis
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
def upload_and_process_video(file_path, max_wait=300):
|
||||
"""Upload video and wait for processing"""
|
||||
myfile = client.files.upload(file=file_path)
|
||||
|
||||
elapsed = 0
|
||||
while myfile.state.name == 'PROCESSING' and elapsed < max_wait:
|
||||
time.sleep(5)
|
||||
myfile = client.files.get(name=myfile.name)
|
||||
elapsed += 5
|
||||
|
||||
if myfile.state.name == 'FAILED':
|
||||
raise ValueError(f'Video processing failed: {myfile.state.name}')
|
||||
|
||||
if myfile.state.name == 'PROCESSING':
|
||||
raise TimeoutError(f'Processing timeout after {max_wait}s')
|
||||
|
||||
return myfile
|
||||
```
|
||||
|
||||
## Cost Optimization
|
||||
|
||||
**Token costs** (Gemini 2.5 Flash at $1/1M):
|
||||
- 1 minute video (default): 18,000 tokens = $0.018
|
||||
- 10 minute video: 180,000 tokens = $0.18
|
||||
- 1 hour video: 1,080,000 tokens = $1.08
|
||||
|
||||
**Strategies**:
|
||||
- Use video clipping for specific segments
|
||||
- Lower FPS for static content
|
||||
- Use low-resolution mode for long videos
|
||||
- Batch related queries on same video
|
||||
- Use context caching for repeated queries
|
||||
|
||||
## Limitations
|
||||
|
||||
- Maximum 6 hours (low-res) or 2 hours (default)
|
||||
- YouTube videos must be public
|
||||
- No live streaming analysis
|
||||
- Files expire after 48 hours
|
||||
- Processing time varies by video length
|
||||
- No real-time processing
|
||||
- Limited to 10 videos per request (2.5+)
|
||||
|
||||
---
|
||||
|
||||
## Related References
|
||||
|
||||
**Current**: Video Analysis
|
||||
|
||||
**Related Capabilities**:
|
||||
- [Video Generation](./video-generation.md) - Creating videos from text/images
|
||||
- [Audio Processing](./audio-processing.md) - Extract and analyze audio tracks
|
||||
- [Image Understanding](./vision-understanding.md) - Analyze individual frames
|
||||
|
||||
**Back to**: [AI Multimodal Skill](../SKILL.md)
|
||||
@@ -0,0 +1,457 @@
|
||||
# Video Generation Reference
|
||||
|
||||
Comprehensive guide for video creation using Veo models via Gemini API.
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
- **Text-to-Video**: Generate 8-second videos from text prompts
|
||||
- **Image-to-Video**: Animate images with text direction
|
||||
- **Video Extension**: Continue previously generated videos
|
||||
- **Frame Control**: Precise camera movements and effects
|
||||
- **Native Audio**: Synchronized audio generation
|
||||
- **Multiple Resolutions**: 720p and 1080p output
|
||||
- **Aspect Ratios**: 16:9, 9:16, 1:1
|
||||
|
||||
## Models
|
||||
|
||||
### Veo 3.1 Preview (Latest)
|
||||
|
||||
**veo-3.1-generate-preview** - Latest with advanced controls
|
||||
- Frame-specific generation
|
||||
- Up to 3 reference images for image-to-video
|
||||
- Video extension capability
|
||||
- Native audio generation
|
||||
- Resolution: 720p, 1080p
|
||||
- Duration: 8 seconds at 24fps
|
||||
- Status: Preview (API may change)
|
||||
- Updated: September 2025
|
||||
|
||||
**veo-3.1-fast-generate-preview** - Speed-optimized
|
||||
- Optimized for business use cases
|
||||
- Programmatic ad creation
|
||||
- Social media content
|
||||
- Same features as standard but faster
|
||||
- Status: Preview
|
||||
- Updated: September 2025
|
||||
|
||||
### Veo 3.0 Stable
|
||||
|
||||
**veo-3.0-generate-001** - Production-ready
|
||||
- Native audio generation
|
||||
- Text-to-video and image-to-video
|
||||
- 720p and 1080p (16:9 only)
|
||||
- 8 seconds at 24fps
|
||||
- Status: Stable
|
||||
- Updated: July 2025
|
||||
|
||||
**veo-3.0-fast-generate-001** - Stable fast variant
|
||||
- Speed-optimized stable version
|
||||
- Same reliability as 3.0
|
||||
- Status: Stable
|
||||
- Updated: July 2025
|
||||
|
||||
## Model Comparison
|
||||
|
||||
| Model | Speed | Features | Audio | Status | Best For |
|
||||
|-------|-------|----------|-------|--------|----------|
|
||||
| veo-3.1-preview | Medium | All | ✓ | Preview | Latest features |
|
||||
| veo-3.1-fast | Fast | All | ✓ | Preview | Business/speed |
|
||||
| veo-3.0-001 | Medium | Standard | ✓ | Stable | Production |
|
||||
| veo-3.0-fast | Fast | Standard | ✓ | Stable | Production/speed |
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Text-to-Video
|
||||
|
||||
```python
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
import os
|
||||
|
||||
client = genai.Client(api_key=os.getenv('GEMINI_API_KEY'))
|
||||
|
||||
# Basic generation
|
||||
response = client.models.generate_video(
|
||||
model='veo-3.1-generate-preview',
|
||||
prompt='A serene beach at sunset with gentle waves rolling onto the shore',
|
||||
config=types.VideoGenerationConfig(
|
||||
resolution='1080p',
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
)
|
||||
|
||||
# Save video
|
||||
with open('output.mp4', 'wb') as f:
|
||||
f.write(response.video.data)
|
||||
```
|
||||
|
||||
### Image-to-Video
|
||||
|
||||
```python
|
||||
import PIL.Image
|
||||
|
||||
# Load reference image
|
||||
ref_image = PIL.Image.open('beach.jpg')
|
||||
|
||||
# Animate the image
|
||||
response = client.models.generate_video(
|
||||
model='veo-3.1-generate-preview',
|
||||
prompt='Camera slowly pans across the scene from left to right',
|
||||
reference_images=[ref_image],
|
||||
config=types.VideoGenerationConfig(
|
||||
resolution='1080p'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Multiple Reference Images
|
||||
|
||||
```python
|
||||
# Use up to 3 reference images for complex scenes
|
||||
img1 = PIL.Image.open('foreground.jpg')
|
||||
img2 = PIL.Image.open('background.jpg')
|
||||
img3 = PIL.Image.open('subject.jpg')
|
||||
|
||||
response = client.models.generate_video(
|
||||
model='veo-3.1-generate-preview',
|
||||
prompt='Combine these elements into a cohesive animated scene',
|
||||
reference_images=[img1, img2, img3],
|
||||
config=types.VideoGenerationConfig(
|
||||
resolution='1080p',
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Video Extension
|
||||
|
||||
```python
|
||||
# Continue from previously generated video
|
||||
previous_video = open('part1.mp4', 'rb').read()
|
||||
|
||||
response = client.models.extend_video(
|
||||
model='veo-3.1-generate-preview',
|
||||
video=previous_video,
|
||||
prompt='The scene transitions to nighttime with stars appearing'
|
||||
)
|
||||
```
|
||||
|
||||
### Frame Control
|
||||
|
||||
```python
|
||||
# Precise camera movements
|
||||
response = client.models.generate_video(
|
||||
model='veo-3.1-generate-preview',
|
||||
prompt='A mountain landscape',
|
||||
config=types.VideoGenerationConfig(
|
||||
resolution='1080p',
|
||||
camera_motion='zoom_in', # Options: zoom_in, zoom_out, pan_left, pan_right, tilt_up, tilt_down, static
|
||||
motion_speed='slow' # Options: slow, medium, fast
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
## Prompt Engineering
|
||||
|
||||
### Effective Video Prompts
|
||||
|
||||
**Structure**:
|
||||
1. **Subject**: What's in the scene
|
||||
2. **Action**: What's happening
|
||||
3. **Camera**: How it's filmed
|
||||
4. **Style**: Visual treatment
|
||||
5. **Timing**: Pacing details
|
||||
|
||||
**Example**:
|
||||
```
|
||||
"A hummingbird [subject] hovers near a red flower, then flies away [action].
|
||||
Slow-motion close-up shot [camera] with vibrant colors and soft focus background [style].
|
||||
Gentle, peaceful pacing [timing]."
|
||||
```
|
||||
|
||||
### Action Verbs
|
||||
|
||||
**Movement**:
|
||||
- "walks", "runs", "flies", "swims", "dances"
|
||||
- "rotates", "spins", "rolls", "bounces"
|
||||
- "emerges", "disappears", "transforms"
|
||||
|
||||
**Camera**:
|
||||
- "zoom in on", "pull back from", "follow"
|
||||
- "orbit around", "track alongside"
|
||||
- "tilt up to reveal", "pan across"
|
||||
|
||||
**Transitions**:
|
||||
- "gradually changes from... to..."
|
||||
- "morphs into", "dissolves into"
|
||||
- "cuts to", "fades to"
|
||||
|
||||
### Timing Control
|
||||
|
||||
```python
|
||||
# Explicit timing in prompt
|
||||
prompt = '''
|
||||
0-2s: Close-up of a seed in soil
|
||||
2-4s: Time-lapse of sprout emerging
|
||||
4-6s: Growing into a small plant
|
||||
6-8s: Zoom out to show garden context
|
||||
'''
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
### Resolution
|
||||
|
||||
```python
|
||||
config = types.VideoGenerationConfig(
|
||||
resolution='1080p' # Options: 720p, 1080p
|
||||
)
|
||||
```
|
||||
|
||||
**Considerations**:
|
||||
- 1080p: Higher quality, longer generation time, larger file
|
||||
- 720p: Faster generation, smaller file, good for drafts
|
||||
|
||||
### Aspect Ratios
|
||||
|
||||
```python
|
||||
config = types.VideoGenerationConfig(
|
||||
aspect_ratio='16:9' # Options: 16:9, 9:16, 1:1
|
||||
)
|
||||
```
|
||||
|
||||
**Use Cases**:
|
||||
- 16:9: Landscape, YouTube, traditional video
|
||||
- 9:16: Mobile, TikTok, Instagram Stories
|
||||
- 1:1: Square, Instagram feed, versatile
|
||||
|
||||
### Audio Control
|
||||
|
||||
```python
|
||||
config = types.VideoGenerationConfig(
|
||||
include_audio=True # Default: True
|
||||
)
|
||||
```
|
||||
|
||||
Native audio is generated automatically and synchronized with video content.
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Prompt Quality
|
||||
|
||||
**Be specific**:
|
||||
- ❌ "A person walking"
|
||||
- ✅ "A young woman in a red coat walking through a park in autumn"
|
||||
|
||||
**Include motion**:
|
||||
- ❌ "A city street"
|
||||
- ✅ "A busy city street with cars passing and people crossing"
|
||||
|
||||
**Specify camera**:
|
||||
- ❌ "A mountain"
|
||||
- ✅ "Aerial drone shot slowly ascending over a snow-capped mountain"
|
||||
|
||||
### 2. Reference Images
|
||||
|
||||
**Quality**:
|
||||
- Use high-resolution images (1080p+)
|
||||
- Clear, well-lit subjects
|
||||
- Minimal motion blur
|
||||
|
||||
**Composition**:
|
||||
- Match desired final aspect ratio
|
||||
- Leave room for motion/movement
|
||||
- Consider camera angle in prompt
|
||||
|
||||
### 3. Performance Optimization
|
||||
|
||||
**Generation Time**:
|
||||
- 720p: ~30-60 seconds
|
||||
- 1080p: ~60-120 seconds
|
||||
- Fast models: 30-50% faster
|
||||
|
||||
**Strategies**:
|
||||
- Use 720p for iteration/drafts
|
||||
- Use fast models for rapid feedback
|
||||
- Batch multiple requests
|
||||
- Use async processing for UI responsiveness
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### 1. Product Demos
|
||||
|
||||
```python
|
||||
response = client.models.generate_video(
|
||||
model='veo-3.0-fast-generate-001',
|
||||
prompt='''
|
||||
Professional product video:
|
||||
- Sleek smartphone rotating on a pedestal
|
||||
- Clean white background with soft shadows
|
||||
- Slow 360-degree rotation
|
||||
- Spotlight highlighting premium design
|
||||
- Modern, minimalist aesthetic
|
||||
''',
|
||||
config=types.VideoGenerationConfig(
|
||||
resolution='1080p',
|
||||
aspect_ratio='1:1'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### 2. Social Media Content
|
||||
|
||||
```python
|
||||
response = client.models.generate_video(
|
||||
model='veo-3.1-fast-generate-preview',
|
||||
prompt='''
|
||||
Trendy social media clip:
|
||||
- Text overlay "NEW ARRIVAL" appears
|
||||
- Fashion product showcase
|
||||
- Quick cuts and dynamic camera
|
||||
- Vibrant colors, high energy
|
||||
- Upbeat pacing
|
||||
''',
|
||||
config=types.VideoGenerationConfig(
|
||||
resolution='1080p',
|
||||
aspect_ratio='9:16' # Mobile
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### 3. Explainer Animations
|
||||
|
||||
```python
|
||||
response = client.models.generate_video(
|
||||
model='veo-3.1-generate-preview',
|
||||
prompt='''
|
||||
Educational animation:
|
||||
- Simple diagram illustrating data flow
|
||||
- Arrows and icons animating in sequence
|
||||
- Clean, clear visual hierarchy
|
||||
- Smooth transitions between steps
|
||||
- Professional corporate style
|
||||
''',
|
||||
config=types.VideoGenerationConfig(
|
||||
resolution='720p',
|
||||
aspect_ratio='16:9'
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
## Safety & Content Policy
|
||||
|
||||
### Safety Settings
|
||||
|
||||
```python
|
||||
config = types.VideoGenerationConfig(
|
||||
safety_settings=[
|
||||
types.SafetySetting(
|
||||
category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
|
||||
threshold=types.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Prohibited Content
|
||||
|
||||
- Violence, gore, harm
|
||||
- Sexually explicit content
|
||||
- Hate speech, harassment
|
||||
- Copyrighted characters/brands
|
||||
- Real people (without consent)
|
||||
- Misleading/deceptive content
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Duration**: Fixed 8 seconds (as of Sept 2025)
|
||||
- **Frame Rate**: 24fps only
|
||||
- **File Size**: ~5-20MB per video
|
||||
- **Generation Time**: 30s-2min depending on resolution
|
||||
- **Reference Images**: Max 3 images
|
||||
- **Preview Status**: API may change (3.1 models)
|
||||
- **Audio**: Cannot upload custom audio (native only)
|
||||
- **No real-time**: Pre-generation required
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Long Generation Times
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
# Track generation progress
|
||||
start = time.time()
|
||||
response = client.models.generate_video(...)
|
||||
duration = time.time() - start
|
||||
print(f"Generated in {duration:.1f}s")
|
||||
```
|
||||
|
||||
**Expected times**:
|
||||
- Fast models + 720p: 30-45s
|
||||
- Standard models + 720p: 45-90s
|
||||
- Fast models + 1080p: 45-60s
|
||||
- Standard models + 1080p: 60-120s
|
||||
|
||||
### Safety Filter Blocking
|
||||
|
||||
```python
|
||||
try:
|
||||
response = client.models.generate_video(...)
|
||||
except Exception as e:
|
||||
if 'safety' in str(e).lower():
|
||||
print("Video blocked by safety filters")
|
||||
# Modify prompt and retry
|
||||
```
|
||||
|
||||
### Quota Exceeded
|
||||
|
||||
```python
|
||||
# Implement exponential backoff
|
||||
import time
|
||||
|
||||
def generate_with_retry(model, prompt, max_retries=3):
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return client.models.generate_video(model=model, prompt=prompt)
|
||||
except Exception as e:
|
||||
if '429' in str(e): # Rate limit
|
||||
wait = 2 ** attempt
|
||||
print(f"Rate limited, waiting {wait}s...")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
raise
|
||||
raise Exception("Max retries exceeded")
|
||||
```
|
||||
|
||||
## Cost Estimation
|
||||
|
||||
**Pricing**: TBD (preview models)
|
||||
|
||||
**Estimated based on compute**:
|
||||
- Fast + 720p: ~$0.05-$0.10 per video
|
||||
- Standard + 1080p: ~$0.15-$0.25 per video
|
||||
|
||||
**Monitor**: https://ai.google.dev/pricing
|
||||
|
||||
## Resources
|
||||
|
||||
- [Veo API Docs](https://ai.google.dev/gemini-api/docs/video)
|
||||
- [Video Generation Guide](https://ai.google.dev/gemini-api/docs/video#model-versions)
|
||||
- [Content Policy](https://ai.google.dev/gemini-api/docs/safety)
|
||||
- [Get API Key](https://aistudio.google.com/apikey)
|
||||
|
||||
---
|
||||
|
||||
## Related References
|
||||
|
||||
**Current**: Video Generation
|
||||
|
||||
**Related Capabilities**:
|
||||
- [Video Analysis](./video-analysis.md) - Understanding existing videos
|
||||
- [Image Generation](./image-generation.md) - Creating static images
|
||||
- [Image Understanding](./vision-understanding.md) - Analyzing reference images
|
||||
|
||||
**Back to**: [AI Multimodal Skill](../SKILL.md)
|
||||
@@ -0,0 +1,492 @@
|
||||
# Vision Understanding Reference
|
||||
|
||||
Comprehensive guide for image analysis, object detection, and visual understanding using Gemini API.
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
- **Captioning**: Generate descriptive text for images
|
||||
- **Classification**: Categorize and identify content
|
||||
- **Visual Q&A**: Answer questions about images
|
||||
- **Object Detection**: Locate objects with bounding boxes (2.0+)
|
||||
- **Segmentation**: Create pixel-level masks (2.5+)
|
||||
- **Multi-image**: Compare up to 3,600 images
|
||||
- **OCR**: Extract text from images
|
||||
- **Document Understanding**: Process PDFs with vision
|
||||
|
||||
## Supported Formats
|
||||
|
||||
- **Images**: PNG, JPEG, WEBP, HEIC, HEIF
|
||||
- **Documents**: PDF (up to 1,000 pages)
|
||||
- **Size Limits**:
|
||||
- Inline: 20MB max total request
|
||||
- File API: 2GB per file
|
||||
- Max images: 3,600 per request
|
||||
|
||||
## Model Selection
|
||||
|
||||
### Gemini 2.5 Series
|
||||
- **gemini-2.5-pro**: Best quality, segmentation + detection
|
||||
- **gemini-2.5-flash**: Fast, efficient, all features
|
||||
- **gemini-2.5-flash-lite**: Lightweight, all features
|
||||
|
||||
### Feature Requirements
|
||||
- **Segmentation**: Requires 2.5+ models
|
||||
- **Object Detection**: Requires 2.0+ models
|
||||
- **Multi-image**: All models (up to 3,600 images)
|
||||
|
||||
## Basic Image Analysis
|
||||
|
||||
### Image Captioning
|
||||
|
||||
```python
|
||||
from google import genai
|
||||
import os
|
||||
|
||||
client = genai.Client(api_key=os.getenv('GEMINI_API_KEY'))
|
||||
|
||||
# Local file
|
||||
with open('image.jpg', 'rb') as f:
|
||||
img_bytes = f.read()
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Describe this image in detail',
|
||||
genai.types.Part.from_bytes(data=img_bytes, mime_type='image/jpeg')
|
||||
]
|
||||
)
|
||||
print(response.text)
|
||||
```
|
||||
|
||||
### Image Classification
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Classify this image. Provide category and confidence level.',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Visual Question Answering
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'How many people are in this image and what are they doing?',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Object Detection (2.5+)
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Detect all objects in this image and provide bounding boxes',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
|
||||
# Returns bounding box coordinates: [ymin, xmin, ymax, xmax]
|
||||
# Normalized to [0, 1000] range
|
||||
```
|
||||
|
||||
### Segmentation (2.5+)
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Create a segmentation mask for all people in this image',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
|
||||
# Returns pixel-level masks for requested objects
|
||||
```
|
||||
|
||||
### Multi-Image Comparison
|
||||
|
||||
```python
|
||||
import PIL.Image
|
||||
|
||||
img1 = PIL.Image.open('photo1.jpg')
|
||||
img2 = PIL.Image.open('photo2.jpg')
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Compare these two images. What are the differences?',
|
||||
img1,
|
||||
img2
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### OCR and Text Extraction
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Extract all visible text from this image',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Input Methods
|
||||
|
||||
### Inline Data (<20MB)
|
||||
|
||||
```python
|
||||
from google.genai import types
|
||||
|
||||
# From file
|
||||
with open('image.jpg', 'rb') as f:
|
||||
img_bytes = f.read()
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Analyze this image',
|
||||
types.Part.from_bytes(data=img_bytes, mime_type='image/jpeg')
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### PIL Image
|
||||
|
||||
```python
|
||||
import PIL.Image
|
||||
|
||||
img = PIL.Image.open('photo.jpg')
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['What is in this image?', img]
|
||||
)
|
||||
```
|
||||
|
||||
### File API (>20MB or Reuse)
|
||||
|
||||
```python
|
||||
# Upload once
|
||||
myfile = client.files.upload(file='large-image.jpg')
|
||||
|
||||
# Use multiple times
|
||||
response1 = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Describe this image', myfile]
|
||||
)
|
||||
|
||||
response2 = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['What colors dominate this image?', myfile]
|
||||
)
|
||||
```
|
||||
|
||||
### URL (Public Images)
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Analyze this image',
|
||||
types.Part.from_uri(
|
||||
uri='https://example.com/image.jpg',
|
||||
mime_type='image/jpeg'
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Token Calculation
|
||||
|
||||
Images consume tokens based on size:
|
||||
|
||||
**Small images** (≤384px both dimensions): 258 tokens
|
||||
|
||||
**Large images**: Tiled into 768×768 chunks, 258 tokens each
|
||||
|
||||
**Formula**:
|
||||
```
|
||||
crop_unit = floor(min(width, height) / 1.5)
|
||||
tiles = (width / crop_unit) × (height / crop_unit)
|
||||
total_tokens = tiles × 258
|
||||
```
|
||||
|
||||
**Examples**:
|
||||
- 256×256: 258 tokens (small)
|
||||
- 512×512: 258 tokens (small)
|
||||
- 960×540: 6 tiles = 1,548 tokens
|
||||
- 1920×1080: 6 tiles = 1,548 tokens
|
||||
- 3840×2160 (4K): 24 tiles = 6,192 tokens
|
||||
|
||||
## Structured Output
|
||||
|
||||
### JSON Schema Output
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
from typing import List
|
||||
|
||||
class ObjectDetection(BaseModel):
|
||||
object_name: str
|
||||
confidence: float
|
||||
bounding_box: List[int] # [ymin, xmin, ymax, xmax]
|
||||
|
||||
class ImageAnalysis(BaseModel):
|
||||
description: str
|
||||
objects: List[ObjectDetection]
|
||||
scene_type: str
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Analyze this image', img_part],
|
||||
config=genai.types.GenerateContentConfig(
|
||||
response_mime_type='application/json',
|
||||
response_schema=ImageAnalysis
|
||||
)
|
||||
)
|
||||
|
||||
result = ImageAnalysis.model_validate_json(response.text)
|
||||
```
|
||||
|
||||
## Multi-Image Analysis
|
||||
|
||||
### Batch Processing
|
||||
|
||||
```python
|
||||
images = [
|
||||
PIL.Image.open(f'image{i}.jpg')
|
||||
for i in range(10)
|
||||
]
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=['Analyze these images and find common themes'] + images
|
||||
)
|
||||
```
|
||||
|
||||
### Image Comparison
|
||||
|
||||
```python
|
||||
before = PIL.Image.open('before.jpg')
|
||||
after = PIL.Image.open('after.jpg')
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Compare before and after. List all visible changes.',
|
||||
before,
|
||||
after
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Visual Search
|
||||
|
||||
```python
|
||||
reference = PIL.Image.open('target.jpg')
|
||||
candidates = [PIL.Image.open(f'option{i}.jpg') for i in range(5)]
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Find which candidate images contain objects similar to the reference',
|
||||
reference
|
||||
] + candidates
|
||||
)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Image Quality
|
||||
|
||||
1. **Resolution**: Use clear, non-blurry images
|
||||
2. **Rotation**: Verify correct orientation
|
||||
3. **Lighting**: Ensure good contrast and lighting
|
||||
4. **Size optimization**: Balance quality vs token cost
|
||||
5. **Format**: JPEG for photos, PNG for graphics
|
||||
|
||||
### Prompt Engineering
|
||||
|
||||
**Specific instructions**:
|
||||
- "Identify all vehicles with their colors and positions"
|
||||
- "Count people wearing blue shirts"
|
||||
- "Extract text from the sign in the top-left corner"
|
||||
|
||||
**Output format**:
|
||||
- "Return results as JSON with fields: category, count, description"
|
||||
- "Format as markdown table"
|
||||
- "List findings as numbered items"
|
||||
|
||||
**Few-shot examples**:
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Example: For an image of a cat on a sofa, respond: "Object: cat, Location: sofa"',
|
||||
'Now analyze this image:',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### File Management
|
||||
|
||||
1. Use File API for images >20MB
|
||||
2. Use File API for repeated queries (saves tokens)
|
||||
3. Files auto-delete after 48 hours
|
||||
4. Clean up manually:
|
||||
```python
|
||||
client.files.delete(name=myfile.name)
|
||||
```
|
||||
|
||||
### Cost Optimization
|
||||
|
||||
**Token-efficient strategies**:
|
||||
- Resize large images before upload
|
||||
- Use File API for repeated queries
|
||||
- Batch multiple images when related
|
||||
- Use appropriate model (Flash vs Pro)
|
||||
|
||||
**Token costs** (Gemini 2.5 Flash at $1/1M):
|
||||
- Small image (258 tokens): $0.000258
|
||||
- HD image (1,548 tokens): $0.001548
|
||||
- 4K image (6,192 tokens): $0.006192
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### 1. Product Analysis
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Analyze this product image:
|
||||
1. Identify the product
|
||||
2. List visible features
|
||||
3. Assess condition
|
||||
4. Estimate value range
|
||||
''',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 2. Screenshot Analysis
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Extract all text and UI elements from this screenshot',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 3. Medical Imaging (Informational Only)
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-pro',
|
||||
contents=[
|
||||
'Describe visible features in this medical image. Note: This is for informational purposes only.',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 4. Chart/Graph Reading
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'Extract data from this chart and format as JSON',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### 5. Scene Understanding
|
||||
|
||||
```python
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
'''Analyze this scene:
|
||||
1. Location type
|
||||
2. Time of day
|
||||
3. Weather conditions
|
||||
4. Activities happening
|
||||
5. Mood/atmosphere
|
||||
''',
|
||||
img_part
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```python
|
||||
import time
|
||||
|
||||
def analyze_image_with_retry(image_path, prompt, max_retries=3):
|
||||
"""Analyze image with exponential backoff retry"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
with open(image_path, 'rb') as f:
|
||||
img_bytes = f.read()
|
||||
|
||||
response = client.models.generate_content(
|
||||
model='gemini-2.5-flash',
|
||||
contents=[
|
||||
prompt,
|
||||
genai.types.Part.from_bytes(
|
||||
data=img_bytes,
|
||||
mime_type='image/jpeg'
|
||||
)
|
||||
]
|
||||
)
|
||||
return response.text
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
wait_time = 2 ** attempt
|
||||
print(f"Retry {attempt + 1} after {wait_time}s: {e}")
|
||||
time.sleep(wait_time)
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
- Maximum 3,600 images per request
|
||||
- OCR accuracy varies with text quality
|
||||
- Object detection requires 2.0+ models
|
||||
- Segmentation requires 2.5+ models
|
||||
- No video frame extraction (use video API)
|
||||
- Regional restrictions on child images (EEA, CH, UK)
|
||||
|
||||
---
|
||||
|
||||
## Related References
|
||||
|
||||
**Current**: Image Understanding
|
||||
|
||||
**Related Capabilities**:
|
||||
- [Image Generation](./image-generation.md) - Create and edit images
|
||||
- [Video Analysis](./video-analysis.md) - Analyze video frames
|
||||
- [Video Generation](./video-generation.md) - Reference images for video generation
|
||||
|
||||
**Back to**: [AI Multimodal Skill](../SKILL.md)
|
||||
Binary file not shown.
Executable
+315
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validate ai-multimodal skill setup and configuration.
|
||||
|
||||
Checks:
|
||||
- API key presence and format
|
||||
- Python dependencies
|
||||
- Centralized resolver availability
|
||||
- Directory structure
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Fix Windows cp1252 encoding: Unicode symbols (✓, ⚠, ✗) can't encode on Windows.
|
||||
# Reconfigure stdout to UTF-8 with replacement (Python 3.7+).
|
||||
if sys.stdout.encoding and sys.stdout.encoding.lower() != "utf-8":
|
||||
if hasattr(sys.stdout, 'reconfigure'):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
if hasattr(sys.stderr, 'reconfigure'):
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
# Color codes for terminal output
|
||||
GREEN = '\033[92m'
|
||||
YELLOW = '\033[93m'
|
||||
RED = '\033[91m'
|
||||
BLUE = '\033[94m'
|
||||
RESET = '\033[0m'
|
||||
BOLD = '\033[1m'
|
||||
|
||||
|
||||
def print_header(text):
|
||||
"""Print section header."""
|
||||
print(f"\n{BOLD}{BLUE}{'='*60}{RESET}")
|
||||
print(f"{BOLD}{BLUE}{text}{RESET}")
|
||||
print(f"{BOLD}{BLUE}{'='*60}{RESET}\n")
|
||||
|
||||
|
||||
def print_success(text):
|
||||
"""Print success message."""
|
||||
print(f"{GREEN}✓ {text}{RESET}")
|
||||
|
||||
|
||||
def print_warning(text):
|
||||
"""Print warning message."""
|
||||
print(f"{YELLOW}⚠ {text}{RESET}")
|
||||
|
||||
|
||||
def print_error(text):
|
||||
"""Print error message."""
|
||||
print(f"{RED}✗ {text}{RESET}")
|
||||
|
||||
|
||||
def print_info(text):
|
||||
"""Print info message."""
|
||||
print(f"{BLUE}ℹ {text}{RESET}")
|
||||
|
||||
|
||||
def check_dependencies():
|
||||
"""Check if required Python packages are installed."""
|
||||
print_header("Checking Python Dependencies")
|
||||
|
||||
dependencies = {
|
||||
'google.genai': 'google-genai',
|
||||
'dotenv': 'python-dotenv',
|
||||
'PIL': 'pillow'
|
||||
}
|
||||
|
||||
missing = []
|
||||
|
||||
for module_name, package_name in dependencies.items():
|
||||
try:
|
||||
__import__(module_name)
|
||||
print_success(f"{package_name} is installed")
|
||||
except ImportError:
|
||||
print_error(f"{package_name} is NOT installed")
|
||||
missing.append(package_name)
|
||||
|
||||
if missing:
|
||||
print_error("\nMissing dependencies detected!")
|
||||
print_info(f"Install with: pip install {' '.join(missing)}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def check_centralized_resolver():
|
||||
"""Check if centralized resolver is available."""
|
||||
print_header("Checking Centralized Resolver")
|
||||
|
||||
claude_root = Path(__file__).parent.parent.parent.parent
|
||||
resolver_path = claude_root / 'scripts' / 'resolve_env.py'
|
||||
|
||||
if resolver_path.exists():
|
||||
print_success(f"Centralized resolver found: {resolver_path}")
|
||||
|
||||
# Try to import it
|
||||
sys.path.insert(0, str(resolver_path.parent))
|
||||
try:
|
||||
from resolve_env import resolve_env
|
||||
print_success("Centralized resolver can be imported")
|
||||
return True
|
||||
except ImportError as e:
|
||||
print_error(f"Centralized resolver exists but cannot be imported: {e}")
|
||||
return False
|
||||
else:
|
||||
print_warning(f"Centralized resolver not found: {resolver_path}")
|
||||
print_info("Skill will use fallback resolution logic")
|
||||
return True # Not critical, fallback works
|
||||
|
||||
|
||||
def find_api_key():
|
||||
"""Find and validate API key using centralized resolver."""
|
||||
print_header("Checking API Key Configuration")
|
||||
|
||||
# Try to use centralized resolver
|
||||
claude_root = Path(__file__).parent.parent.parent.parent
|
||||
sys.path.insert(0, str(claude_root / 'scripts'))
|
||||
try:
|
||||
from resolve_env import resolve_env
|
||||
|
||||
print_info("Using centralized resolver...")
|
||||
api_key = resolve_env('GEMINI_API_KEY', skill='ai-multimodal')
|
||||
|
||||
if api_key:
|
||||
print_success("API key found via centralized resolver")
|
||||
print_info(f"Key preview: {api_key[:20]}...{api_key[-4:]}")
|
||||
|
||||
# Show hierarchy
|
||||
print_info("\nTo see where the key was found, run:")
|
||||
print_info("python ~/.claude/scripts/resolve_env.py GEMINI_API_KEY --skill ai-multimodal --verbose")
|
||||
|
||||
return api_key
|
||||
else:
|
||||
print_error("API key not found in any location")
|
||||
return None
|
||||
|
||||
except ImportError:
|
||||
print_warning("Centralized resolver not available, using fallback")
|
||||
|
||||
# Fallback: check environment
|
||||
api_key = os.getenv('GEMINI_API_KEY')
|
||||
if api_key:
|
||||
print_success("API key found in process.env")
|
||||
print_info(f"Key preview: {api_key[:20]}...{api_key[-4:]}")
|
||||
return api_key
|
||||
else:
|
||||
print_error("API key not found")
|
||||
return None
|
||||
|
||||
|
||||
def validate_api_key_format(api_key):
|
||||
"""Basic validation of API key format."""
|
||||
if not api_key:
|
||||
return False
|
||||
|
||||
# Google AI Studio keys typically start with 'AIza'
|
||||
if api_key.startswith('AIza'):
|
||||
print_success("API key format looks valid (Google AI Studio)")
|
||||
return True
|
||||
elif len(api_key) > 20:
|
||||
print_warning("API key format not recognized (may be Vertex AI or custom)")
|
||||
return True
|
||||
else:
|
||||
print_error("API key format looks invalid (too short)")
|
||||
return False
|
||||
|
||||
|
||||
def test_api_connection(api_key):
|
||||
"""Test API connection with a simple request."""
|
||||
print_header("Testing API Connection")
|
||||
|
||||
try:
|
||||
from google import genai
|
||||
|
||||
print_info("Initializing Gemini client...")
|
||||
client = genai.Client(api_key=api_key)
|
||||
|
||||
print_info("Fetching available models...")
|
||||
# List models to verify API key works
|
||||
models = list(client.models.list())
|
||||
|
||||
print_success(f"API connection successful! Found {len(models)} available models")
|
||||
|
||||
# Show some available models
|
||||
print_info("\nSample available models:")
|
||||
for model in models[:5]:
|
||||
print(f" - {model.name}")
|
||||
|
||||
return True
|
||||
|
||||
except ImportError:
|
||||
print_error("google-genai package not installed")
|
||||
return False
|
||||
except Exception as e:
|
||||
print_error(f"API connection failed: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def check_directory_structure():
|
||||
"""Verify skill directory structure."""
|
||||
print_header("Checking Directory Structure")
|
||||
|
||||
script_dir = Path(__file__).parent
|
||||
skill_dir = script_dir.parent
|
||||
|
||||
required_files = [
|
||||
('SKILL.md', skill_dir / 'SKILL.md'),
|
||||
('.env.example', skill_dir / '.env.example'),
|
||||
('gemini_batch_process.py', script_dir / 'gemini_batch_process.py'),
|
||||
]
|
||||
|
||||
all_exist = True
|
||||
|
||||
for name, path in required_files:
|
||||
if path.exists():
|
||||
print_success(f"{name} exists")
|
||||
else:
|
||||
print_error(f"{name} NOT found at {path}")
|
||||
all_exist = False
|
||||
|
||||
return all_exist
|
||||
|
||||
|
||||
def provide_setup_instructions():
|
||||
"""Provide setup instructions if configuration is incomplete."""
|
||||
print_header("Setup Instructions")
|
||||
|
||||
print_info("To configure the ai-multimodal skill:")
|
||||
print("\n1. Get a Gemini API key:")
|
||||
print(" → Visit: https://aistudio.google.com/apikey")
|
||||
|
||||
print("\n2. Configure the API key (choose one method):")
|
||||
|
||||
print(f"\n Option A: User global config (recommended)")
|
||||
print(f" $ echo 'GEMINI_API_KEY=your-api-key-here' >> ~/.claude/.env")
|
||||
|
||||
script_dir = Path(__file__).parent
|
||||
skill_dir = script_dir.parent
|
||||
|
||||
print(f"\n Option B: Skill-specific config")
|
||||
print(f" $ cd {skill_dir}")
|
||||
print(f" $ cp .env.example .env")
|
||||
print(f" $ # Edit .env and add your API key")
|
||||
|
||||
print(f"\n Option C: Runtime environment (temporary)")
|
||||
print(f" $ export GEMINI_API_KEY='your-api-key-here'")
|
||||
|
||||
print("\n3. Verify setup:")
|
||||
print(f" $ python {Path(__file__)}")
|
||||
|
||||
print("\n4. Debug if needed:")
|
||||
print(f" $ python ~/.claude/scripts/resolve_env.py --show-hierarchy --skill ai-multimodal")
|
||||
print(f" $ python ~/.claude/scripts/resolve_env.py GEMINI_API_KEY --skill ai-multimodal --verbose")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all setup checks."""
|
||||
print(f"\n{BOLD}AI Multimodal Skill - Setup Checker{RESET}")
|
||||
|
||||
all_passed = True
|
||||
|
||||
# Check directory structure
|
||||
if not check_directory_structure():
|
||||
all_passed = False
|
||||
|
||||
# Check centralized resolver
|
||||
check_centralized_resolver()
|
||||
|
||||
# Check dependencies
|
||||
if not check_dependencies():
|
||||
all_passed = False
|
||||
provide_setup_instructions()
|
||||
sys.exit(1)
|
||||
|
||||
# Check API key
|
||||
api_key = find_api_key()
|
||||
|
||||
if not api_key:
|
||||
print_error("\n❌ GEMINI_API_KEY not found in any location")
|
||||
all_passed = False
|
||||
provide_setup_instructions()
|
||||
sys.exit(1)
|
||||
|
||||
# Validate API key format
|
||||
if not validate_api_key_format(api_key):
|
||||
all_passed = False
|
||||
|
||||
# Test API connection
|
||||
if not test_api_connection(api_key):
|
||||
all_passed = False
|
||||
|
||||
# Final summary
|
||||
print_header("Setup Summary")
|
||||
|
||||
if all_passed:
|
||||
print_success("✅ All checks passed! The ai-multimodal skill is ready to use.")
|
||||
print_info("\nNext steps:")
|
||||
print(" • Read SKILL.md for usage examples")
|
||||
print(" • Try: python scripts/gemini_batch_process.py --help")
|
||||
print("\nImage generation models:")
|
||||
print(" • gemini-2.5-flash-image - Nano Banana Flash (DEFAULT - fast)")
|
||||
print(" • imagen-4.0-generate-001 - Imagen 4 (alternative - production)")
|
||||
print(" • gemini-3-pro-image-preview - Nano Banana Pro (4K text, reasoning)")
|
||||
print("\nExample (uses default model):")
|
||||
print(" python scripts/gemini_batch_process.py --task generate \\")
|
||||
print(" --prompt 'A sunset over mountains' --aspect-ratio 16:9 --size 2K")
|
||||
else:
|
||||
print_error("❌ Some checks failed. Please fix the issues above.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+395
@@ -0,0 +1,395 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Convert documents to Markdown using Gemini API.
|
||||
|
||||
Supports all document types:
|
||||
- PDF documents (native vision processing)
|
||||
- Images (JPEG, PNG, WEBP, HEIC)
|
||||
- Office documents (DOCX, XLSX, PPTX)
|
||||
- HTML, TXT, and other text formats
|
||||
|
||||
Features:
|
||||
- Converts to clean markdown format
|
||||
- Preserves structure, tables, and formatting
|
||||
- Extracts text from images and scanned documents
|
||||
- Batch conversion support
|
||||
- Saves to docs/assets/document-extraction.md by default
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
try:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
except ImportError:
|
||||
print("Error: google-genai package not installed")
|
||||
print("Install with: pip install google-genai")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
load_dotenv = None
|
||||
|
||||
|
||||
def find_api_key() -> Optional[str]:
|
||||
"""Find Gemini API key using correct priority order.
|
||||
|
||||
Priority order (highest to lowest):
|
||||
1. process.env (runtime environment variables)
|
||||
2. .claude/skills/ai-multimodal/.env (skill-specific config)
|
||||
3. .claude/skills/.env (shared skills config)
|
||||
4. .claude/.env (Claude global config)
|
||||
"""
|
||||
# Priority 1: Already in process.env (highest)
|
||||
api_key = os.getenv('GEMINI_API_KEY')
|
||||
if api_key:
|
||||
return api_key
|
||||
|
||||
# Load .env files if dotenv available
|
||||
if load_dotenv:
|
||||
# Determine base paths
|
||||
script_dir = Path(__file__).parent
|
||||
skill_dir = script_dir.parent # .claude/skills/ai-multimodal
|
||||
skills_dir = skill_dir.parent # .claude/skills
|
||||
claude_dir = skills_dir.parent # .claude
|
||||
|
||||
# Priority 2: Skill-specific .env
|
||||
env_file = skill_dir / '.env'
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
api_key = os.getenv('GEMINI_API_KEY')
|
||||
if api_key:
|
||||
return api_key
|
||||
|
||||
# Priority 3: Shared skills .env
|
||||
env_file = skills_dir / '.env'
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
api_key = os.getenv('GEMINI_API_KEY')
|
||||
if api_key:
|
||||
return api_key
|
||||
|
||||
# Priority 4: Claude global .env
|
||||
env_file = claude_dir / '.env'
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
api_key = os.getenv('GEMINI_API_KEY')
|
||||
if api_key:
|
||||
return api_key
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_project_root() -> Path:
|
||||
"""Find project root directory."""
|
||||
script_dir = Path(__file__).parent
|
||||
|
||||
# Look for .git or .claude directory
|
||||
for parent in [script_dir] + list(script_dir.parents):
|
||||
if (parent / '.git').exists() or (parent / '.claude').exists():
|
||||
return parent
|
||||
|
||||
return script_dir
|
||||
|
||||
|
||||
def get_mime_type(file_path: str) -> str:
|
||||
"""Determine MIME type from file extension."""
|
||||
ext = Path(file_path).suffix.lower()
|
||||
|
||||
mime_types = {
|
||||
# Documents
|
||||
'.pdf': 'application/pdf',
|
||||
'.txt': 'text/plain',
|
||||
'.html': 'text/html',
|
||||
'.htm': 'text/html',
|
||||
'.md': 'text/markdown',
|
||||
'.csv': 'text/csv',
|
||||
# Images
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.webp': 'image/webp',
|
||||
'.heic': 'image/heic',
|
||||
'.heif': 'image/heif',
|
||||
# Office (need to be uploaded as binary)
|
||||
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
}
|
||||
|
||||
return mime_types.get(ext, 'application/octet-stream')
|
||||
|
||||
|
||||
def upload_file(client: genai.Client, file_path: str, verbose: bool = False) -> Any:
|
||||
"""Upload file to Gemini File API."""
|
||||
if verbose:
|
||||
print(f"Uploading {file_path}...")
|
||||
|
||||
myfile = client.files.upload(file=file_path)
|
||||
|
||||
# Wait for processing if needed
|
||||
max_wait = 300 # 5 minutes
|
||||
elapsed = 0
|
||||
while myfile.state.name == 'PROCESSING' and elapsed < max_wait:
|
||||
time.sleep(2)
|
||||
myfile = client.files.get(name=myfile.name)
|
||||
elapsed += 2
|
||||
if verbose and elapsed % 10 == 0:
|
||||
print(f" Processing... {elapsed}s")
|
||||
|
||||
if myfile.state.name == 'FAILED':
|
||||
raise ValueError(f"File processing failed: {file_path}")
|
||||
|
||||
if myfile.state.name == 'PROCESSING':
|
||||
raise TimeoutError(f"Processing timeout after {max_wait}s: {file_path}")
|
||||
|
||||
if verbose:
|
||||
print(f" Uploaded: {myfile.name}")
|
||||
|
||||
return myfile
|
||||
|
||||
|
||||
def convert_to_markdown(
|
||||
client: genai.Client,
|
||||
file_path: str,
|
||||
model: str = 'gemini-2.5-flash',
|
||||
custom_prompt: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
max_retries: int = 3
|
||||
) -> Dict[str, Any]:
|
||||
"""Convert a document to markdown using Gemini."""
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
file_path_obj = Path(file_path)
|
||||
file_size = file_path_obj.stat().st_size
|
||||
use_file_api = file_size > 20 * 1024 * 1024 # >20MB
|
||||
|
||||
# Default prompt for markdown conversion
|
||||
if custom_prompt:
|
||||
prompt = custom_prompt
|
||||
else:
|
||||
prompt = """Convert this document to clean, well-formatted Markdown.
|
||||
|
||||
Requirements:
|
||||
- Preserve all content, structure, and formatting
|
||||
- Convert tables to markdown table format
|
||||
- Maintain heading hierarchy (# ## ### etc)
|
||||
- Preserve lists, code blocks, and quotes
|
||||
- Extract text from images if present
|
||||
- Keep formatting consistent and readable
|
||||
|
||||
Output only the markdown content without any preamble or explanation."""
|
||||
|
||||
# Upload or inline the file
|
||||
if use_file_api:
|
||||
myfile = upload_file(client, str(file_path), verbose)
|
||||
content = [prompt, myfile]
|
||||
else:
|
||||
with open(file_path, 'rb') as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
mime_type = get_mime_type(str(file_path))
|
||||
content = [
|
||||
prompt,
|
||||
types.Part.from_bytes(data=file_bytes, mime_type=mime_type)
|
||||
]
|
||||
|
||||
# Generate markdown
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=content
|
||||
)
|
||||
|
||||
markdown_content = response.text if hasattr(response, 'text') else ''
|
||||
|
||||
return {
|
||||
'file': str(file_path),
|
||||
'status': 'success',
|
||||
'markdown': markdown_content
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
return {
|
||||
'file': str(file_path),
|
||||
'status': 'error',
|
||||
'error': str(e),
|
||||
'markdown': None
|
||||
}
|
||||
|
||||
wait_time = 2 ** attempt
|
||||
if verbose:
|
||||
print(f" Retry {attempt + 1} after {wait_time}s: {e}")
|
||||
time.sleep(wait_time)
|
||||
|
||||
|
||||
def batch_convert(
|
||||
files: List[str],
|
||||
output_file: Optional[str] = None,
|
||||
auto_name: bool = False,
|
||||
model: str = 'gemini-2.5-flash',
|
||||
custom_prompt: Optional[str] = None,
|
||||
verbose: bool = False
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Batch convert multiple files to markdown."""
|
||||
|
||||
api_key = find_api_key()
|
||||
if not api_key:
|
||||
print("Error: GEMINI_API_KEY not found")
|
||||
print("Set via: export GEMINI_API_KEY='your-key'")
|
||||
print("Or create .env file with: GEMINI_API_KEY=your-key")
|
||||
sys.exit(1)
|
||||
|
||||
client = genai.Client(api_key=api_key)
|
||||
results = []
|
||||
|
||||
# Determine output path
|
||||
if not output_file:
|
||||
project_root = find_project_root()
|
||||
output_dir = project_root / 'docs' / 'assets'
|
||||
|
||||
if auto_name and len(files) == 1:
|
||||
# Auto-generate meaningful filename from input
|
||||
input_path = Path(files[0])
|
||||
base_name = input_path.stem
|
||||
output_file = str(output_dir / f"{base_name}-extraction.md")
|
||||
else:
|
||||
output_file = str(output_dir / 'document-extraction.md')
|
||||
|
||||
output_path = Path(output_file)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Process each file
|
||||
for i, file_path in enumerate(files, 1):
|
||||
if verbose:
|
||||
print(f"\n[{i}/{len(files)}] Converting: {file_path}")
|
||||
|
||||
result = convert_to_markdown(
|
||||
client=client,
|
||||
file_path=file_path,
|
||||
model=model,
|
||||
custom_prompt=custom_prompt,
|
||||
verbose=verbose
|
||||
)
|
||||
|
||||
results.append(result)
|
||||
|
||||
if verbose:
|
||||
status = result.get('status', 'unknown')
|
||||
print(f" Status: {status}")
|
||||
|
||||
# Save combined markdown
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write("# Document Extraction Results\n\n")
|
||||
f.write(f"Converted {len(files)} document(s) to markdown.\n\n")
|
||||
f.write("---\n\n")
|
||||
|
||||
for result in results:
|
||||
f.write(f"## {Path(result['file']).name}\n\n")
|
||||
|
||||
if result['status'] == 'success' and result.get('markdown'):
|
||||
f.write(result['markdown'])
|
||||
f.write("\n\n")
|
||||
elif result['status'] == 'success':
|
||||
f.write("**Note**: Conversion succeeded but no content was returned.\n\n")
|
||||
else:
|
||||
f.write(f"**Error**: {result.get('error', 'Unknown error')}\n\n")
|
||||
|
||||
f.write("---\n\n")
|
||||
|
||||
if verbose or True: # Always show output location
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Converted: {len(results)} file(s)")
|
||||
print(f"Success: {sum(1 for r in results if r['status'] == 'success')}")
|
||||
print(f"Failed: {sum(1 for r in results if r['status'] == 'error')}")
|
||||
print(f"Output saved to: {output_path}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Convert documents to Markdown using Gemini API',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Convert single PDF to markdown (default name)
|
||||
%(prog)s --input document.pdf
|
||||
|
||||
# Auto-generate meaningful filename
|
||||
%(prog)s --input testpdf.pdf --auto-name
|
||||
# Output: docs/assets/testpdf-extraction.md
|
||||
|
||||
# Convert multiple files
|
||||
%(prog)s --input doc1.pdf doc2.docx image.png
|
||||
|
||||
# Specify custom output location
|
||||
%(prog)s --input document.pdf --output ./output.md
|
||||
|
||||
# Use custom prompt
|
||||
%(prog)s --input document.pdf --prompt "Extract only the tables as markdown"
|
||||
|
||||
# Batch convert directory
|
||||
%(prog)s --input ./documents/*.pdf --verbose
|
||||
|
||||
Supported formats:
|
||||
- PDF documents (up to 1,000 pages)
|
||||
- Images (JPEG, PNG, WEBP, HEIC)
|
||||
- Office documents (DOCX, XLSX, PPTX)
|
||||
- Text formats (TXT, HTML, Markdown, CSV)
|
||||
|
||||
Default output: <project-root>/docs/assets/document-extraction.md
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('--input', '-i', nargs='+', required=True,
|
||||
help='Input file(s) to convert')
|
||||
parser.add_argument('--output', '-o',
|
||||
help='Output markdown file (default: docs/assets/document-extraction.md)')
|
||||
parser.add_argument('--auto-name', '-a', action='store_true',
|
||||
help='Auto-generate meaningful output filename from input (e.g., document.pdf -> document-extraction.md)')
|
||||
parser.add_argument('--model', default='gemini-2.5-flash',
|
||||
help='Gemini model to use (default: gemini-2.5-flash)')
|
||||
parser.add_argument('--prompt', '-p',
|
||||
help='Custom prompt for conversion')
|
||||
parser.add_argument('--verbose', '-v', action='store_true',
|
||||
help='Verbose output')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate input files
|
||||
files = []
|
||||
for file_pattern in args.input:
|
||||
file_path = Path(file_pattern)
|
||||
if file_path.exists() and file_path.is_file():
|
||||
files.append(str(file_path))
|
||||
else:
|
||||
# Try glob pattern
|
||||
import glob
|
||||
matched = glob.glob(file_pattern)
|
||||
files.extend([f for f in matched if Path(f).is_file()])
|
||||
|
||||
if not files:
|
||||
print("Error: No valid input files found")
|
||||
sys.exit(1)
|
||||
|
||||
# Convert files
|
||||
batch_convert(
|
||||
files=files,
|
||||
output_file=args.output,
|
||||
auto_name=args.auto_name,
|
||||
model=args.model,
|
||||
custom_prompt=args.prompt,
|
||||
verbose=args.verbose
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+1185
File diff suppressed because it is too large.
Load diff
+506
@@ -0,0 +1,506 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Optimize media files for Gemini API processing.
|
||||
|
||||
Features:
|
||||
- Compress videos/audio for size limits
|
||||
- Resize images appropriately
|
||||
- Split long videos into chunks
|
||||
- Format conversion
|
||||
- Quality vs size optimization
|
||||
- Validation before upload
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
load_dotenv = None
|
||||
|
||||
|
||||
def load_env_files():
|
||||
"""Load .env files in correct priority order.
|
||||
|
||||
Priority order (highest to lowest):
|
||||
1. process.env (runtime environment variables)
|
||||
2. .claude/skills/ai-multimodal/.env (skill-specific config)
|
||||
3. .claude/skills/.env (shared skills config)
|
||||
4. .claude/.env (Claude global config)
|
||||
"""
|
||||
if not load_dotenv:
|
||||
return
|
||||
|
||||
# Determine base paths
|
||||
script_dir = Path(__file__).parent
|
||||
skill_dir = script_dir.parent # .claude/skills/ai-multimodal
|
||||
skills_dir = skill_dir.parent # .claude/skills
|
||||
claude_dir = skills_dir.parent # .claude
|
||||
|
||||
# Priority 2: Skill-specific .env
|
||||
env_file = skill_dir / '.env'
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
|
||||
# Priority 3: Shared skills .env
|
||||
env_file = skills_dir / '.env'
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
|
||||
# Priority 4: Claude global .env
|
||||
env_file = claude_dir / '.env'
|
||||
if env_file.exists():
|
||||
load_dotenv(env_file)
|
||||
|
||||
|
||||
# Load environment variables at module level
|
||||
load_env_files()
|
||||
|
||||
|
||||
def check_ffmpeg() -> bool:
|
||||
"""Check if ffmpeg is installed."""
|
||||
try:
|
||||
subprocess.run(['ffmpeg', '-version'],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=True)
|
||||
return True
|
||||
except (subprocess.CalledProcessError, FileNotFoundError, Exception):
|
||||
return False
|
||||
|
||||
|
||||
def get_media_info(file_path: str) -> Dict[str, Any]:
|
||||
"""Get media file information using ffprobe."""
|
||||
if not check_ffmpeg():
|
||||
return {}
|
||||
|
||||
try:
|
||||
cmd = [
|
||||
'ffprobe',
|
||||
'-v', 'quiet',
|
||||
'-print_format', 'json',
|
||||
'-show_format',
|
||||
'-show_streams',
|
||||
file_path
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
|
||||
data = json.loads(result.stdout)
|
||||
|
||||
info = {
|
||||
'size': int(data['format'].get('size', 0)),
|
||||
'duration': float(data['format'].get('duration', 0)),
|
||||
'bit_rate': int(data['format'].get('bit_rate', 0)),
|
||||
}
|
||||
|
||||
# Get video/audio specific info
|
||||
for stream in data.get('streams', []):
|
||||
if stream['codec_type'] == 'video':
|
||||
info['width'] = stream.get('width', 0)
|
||||
info['height'] = stream.get('height', 0)
|
||||
info['fps'] = eval(stream.get('r_frame_rate', '0/1'))
|
||||
elif stream['codec_type'] == 'audio':
|
||||
info['sample_rate'] = int(stream.get('sample_rate', 0))
|
||||
info['channels'] = stream.get('channels', 0)
|
||||
|
||||
return info
|
||||
|
||||
except (subprocess.CalledProcessError, json.JSONDecodeError, Exception):
|
||||
return {}
|
||||
|
||||
|
||||
def optimize_video(
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
target_size_mb: Optional[int] = None,
|
||||
max_duration: Optional[int] = None,
|
||||
quality: int = 23,
|
||||
resolution: Optional[str] = None,
|
||||
verbose: bool = False
|
||||
) -> bool:
|
||||
"""Optimize video file for Gemini API."""
|
||||
if not check_ffmpeg():
|
||||
print("Error: ffmpeg not installed")
|
||||
print("Install: apt-get install ffmpeg (Linux) or brew install ffmpeg (Mac)")
|
||||
return False
|
||||
|
||||
info = get_media_info(input_path)
|
||||
if not info:
|
||||
print(f"Error: Could not read media info from {input_path}")
|
||||
return False
|
||||
|
||||
if verbose:
|
||||
print(f"Input: {Path(input_path).name}")
|
||||
print(f" Size: {info['size'] / (1024*1024):.2f} MB")
|
||||
print(f" Duration: {info['duration']:.2f}s")
|
||||
if 'width' in info:
|
||||
print(f" Resolution: {info['width']}x{info['height']}")
|
||||
print(f" Bit rate: {info['bit_rate'] / 1000:.0f} kbps")
|
||||
|
||||
# Build ffmpeg command
|
||||
cmd = ['ffmpeg', '-i', input_path, '-y']
|
||||
|
||||
# Video codec
|
||||
cmd.extend(['-c:v', 'libx264', '-crf', str(quality)])
|
||||
|
||||
# Resolution
|
||||
if resolution:
|
||||
cmd.extend(['-vf', f'scale={resolution}'])
|
||||
elif 'width' in info and info['width'] > 1920:
|
||||
cmd.extend(['-vf', 'scale=1920:-2']) # Max 1080p
|
||||
|
||||
# Audio codec
|
||||
cmd.extend(['-c:a', 'aac', '-b:a', '128k', '-ac', '2'])
|
||||
|
||||
# Duration limit
|
||||
if max_duration and info['duration'] > max_duration:
|
||||
cmd.extend(['-t', str(max_duration)])
|
||||
|
||||
# Target size (rough estimate using bitrate)
|
||||
if target_size_mb:
|
||||
target_bits = target_size_mb * 8 * 1024 * 1024
|
||||
duration = min(info['duration'], max_duration) if max_duration else info['duration']
|
||||
target_bitrate = int(target_bits / duration)
|
||||
# Reserve some for audio (128kbps)
|
||||
video_bitrate = max(target_bitrate - 128000, 500000)
|
||||
cmd.extend(['-b:v', str(video_bitrate)])
|
||||
|
||||
cmd.append(output_path)
|
||||
|
||||
if verbose:
|
||||
print(f"\nOptimizing...")
|
||||
print(f" Command: {' '.join(cmd)}")
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=not verbose)
|
||||
|
||||
# Check output
|
||||
output_info = get_media_info(output_path)
|
||||
if output_info and verbose:
|
||||
print(f"\nOutput: {Path(output_path).name}")
|
||||
print(f" Size: {output_info['size'] / (1024*1024):.2f} MB")
|
||||
print(f" Duration: {output_info['duration']:.2f}s")
|
||||
if 'width' in output_info:
|
||||
print(f" Resolution: {output_info['width']}x{output_info['height']}")
|
||||
compression = (1 - output_info['size'] / info['size']) * 100
|
||||
print(f" Compression: {compression:.1f}%")
|
||||
|
||||
return True
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error optimizing video: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def optimize_audio(
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
target_size_mb: Optional[int] = None,
|
||||
bitrate: str = '64k',
|
||||
sample_rate: int = 16000,
|
||||
verbose: bool = False
|
||||
) -> bool:
|
||||
"""Optimize audio file for Gemini API."""
|
||||
if not check_ffmpeg():
|
||||
print("Error: ffmpeg not installed")
|
||||
return False
|
||||
|
||||
info = get_media_info(input_path)
|
||||
if not info:
|
||||
print(f"Error: Could not read media info from {input_path}")
|
||||
return False
|
||||
|
||||
if verbose:
|
||||
print(f"Input: {Path(input_path).name}")
|
||||
print(f" Size: {info['size'] / (1024*1024):.2f} MB")
|
||||
print(f" Duration: {info['duration']:.2f}s")
|
||||
|
||||
# Build command
|
||||
cmd = [
|
||||
'ffmpeg', '-i', input_path, '-y',
|
||||
'-c:a', 'aac',
|
||||
'-b:a', bitrate,
|
||||
'-ar', str(sample_rate),
|
||||
'-ac', '1', # Mono (Gemini uses mono anyway)
|
||||
output_path
|
||||
]
|
||||
|
||||
if verbose:
|
||||
print(f"\nOptimizing...")
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=not verbose)
|
||||
|
||||
output_info = get_media_info(output_path)
|
||||
if output_info and verbose:
|
||||
print(f"\nOutput: {Path(output_path).name}")
|
||||
print(f" Size: {output_info['size'] / (1024*1024):.2f} MB")
|
||||
compression = (1 - output_info['size'] / info['size']) * 100
|
||||
print(f" Compression: {compression:.1f}%")
|
||||
|
||||
return True
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error optimizing audio: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def optimize_image(
|
||||
input_path: str,
|
||||
output_path: str,
|
||||
max_width: int = 1920,
|
||||
quality: int = 85,
|
||||
verbose: bool = False
|
||||
) -> bool:
|
||||
"""Optimize image file for Gemini API."""
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
print("Error: Pillow not installed")
|
||||
print("Install with: pip install pillow")
|
||||
return False
|
||||
|
||||
try:
|
||||
img = Image.open(input_path)
|
||||
|
||||
if verbose:
|
||||
print(f"Input: {Path(input_path).name}")
|
||||
print(f" Size: {Path(input_path).stat().st_size / 1024:.2f} KB")
|
||||
print(f" Resolution: {img.width}x{img.height}")
|
||||
|
||||
# Resize if needed
|
||||
if img.width > max_width:
|
||||
ratio = max_width / img.width
|
||||
new_height = int(img.height * ratio)
|
||||
img = img.resize((max_width, new_height), Image.Resampling.LANCZOS)
|
||||
if verbose:
|
||||
print(f" Resized to: {img.width}x{img.height}")
|
||||
|
||||
# Convert RGBA to RGB if saving as JPEG
|
||||
if output_path.lower().endswith('.jpg') or output_path.lower().endswith('.jpeg'):
|
||||
if img.mode == 'RGBA':
|
||||
rgb_img = Image.new('RGB', img.size, (255, 255, 255))
|
||||
rgb_img.paste(img, mask=img.split()[3])
|
||||
img = rgb_img
|
||||
|
||||
# Save
|
||||
img.save(output_path, quality=quality, optimize=True)
|
||||
|
||||
if verbose:
|
||||
print(f"\nOutput: {Path(output_path).name}")
|
||||
print(f" Size: {Path(output_path).stat().st_size / 1024:.2f} KB")
|
||||
compression = (1 - Path(output_path).stat().st_size / Path(input_path).stat().st_size) * 100
|
||||
print(f" Compression: {compression:.1f}%")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error optimizing image: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def split_video(
|
||||
input_path: str,
|
||||
output_dir: str,
|
||||
chunk_duration: int = 3600,
|
||||
verbose: bool = False
|
||||
) -> List[str]:
|
||||
"""Split long video into chunks."""
|
||||
if not check_ffmpeg():
|
||||
print("Error: ffmpeg not installed")
|
||||
return []
|
||||
|
||||
info = get_media_info(input_path)
|
||||
if not info:
|
||||
return []
|
||||
|
||||
total_duration = info['duration']
|
||||
num_chunks = int(total_duration / chunk_duration) + 1
|
||||
|
||||
if num_chunks == 1:
|
||||
if verbose:
|
||||
print("Video is short enough, no splitting needed")
|
||||
return [input_path]
|
||||
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
output_files = []
|
||||
|
||||
for i in range(num_chunks):
|
||||
start_time = i * chunk_duration
|
||||
output_file = Path(output_dir) / f"{Path(input_path).stem}_chunk_{i+1}.mp4"
|
||||
|
||||
cmd = [
|
||||
'ffmpeg', '-i', input_path, '-y',
|
||||
'-ss', str(start_time),
|
||||
'-t', str(chunk_duration),
|
||||
'-c', 'copy',
|
||||
str(output_file)
|
||||
]
|
||||
|
||||
if verbose:
|
||||
print(f"Creating chunk {i+1}/{num_chunks}...")
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=not verbose)
|
||||
output_files.append(str(output_file))
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error creating chunk {i+1}: {e}")
|
||||
|
||||
return output_files
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Optimize media files for Gemini API',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Optimize video to 100MB
|
||||
%(prog)s --input video.mp4 --output optimized.mp4 --target-size 100
|
||||
|
||||
# Optimize audio
|
||||
%(prog)s --input audio.mp3 --output optimized.m4a --bitrate 64k
|
||||
|
||||
# Resize image
|
||||
%(prog)s --input image.jpg --output resized.jpg --max-width 1920
|
||||
|
||||
# Split long video
|
||||
%(prog)s --input long-video.mp4 --split --chunk-duration 3600 --output-dir ./chunks
|
||||
|
||||
# Batch optimize directory
|
||||
%(prog)s --input-dir ./videos --output-dir ./optimized --quality 85
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('--input', help='Input file')
|
||||
parser.add_argument('--output', help='Output file')
|
||||
parser.add_argument('--input-dir', help='Input directory for batch processing')
|
||||
parser.add_argument('--output-dir', help='Output directory for batch processing')
|
||||
parser.add_argument('--target-size', type=int, help='Target size in MB')
|
||||
parser.add_argument('--quality', type=int, default=85,
|
||||
help='Quality (video: 0-51 CRF, image: 1-100) (default: 85)')
|
||||
parser.add_argument('--max-width', type=int, default=1920,
|
||||
help='Max image width (default: 1920)')
|
||||
parser.add_argument('--bitrate', default='64k',
|
||||
help='Audio bitrate (default: 64k)')
|
||||
parser.add_argument('--resolution', help='Video resolution (e.g., 1920x1080)')
|
||||
parser.add_argument('--split', action='store_true', help='Split long video into chunks')
|
||||
parser.add_argument('--chunk-duration', type=int, default=3600,
|
||||
help='Chunk duration in seconds (default: 3600 = 1 hour)')
|
||||
parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate arguments
|
||||
if not args.input and not args.input_dir:
|
||||
parser.error("Either --input or --input-dir required")
|
||||
|
||||
# Single file processing
|
||||
if args.input:
|
||||
input_path = Path(args.input)
|
||||
if not input_path.exists():
|
||||
print(f"Error: Input file not found: {input_path}")
|
||||
sys.exit(1)
|
||||
|
||||
if args.split:
|
||||
output_dir = args.output_dir or './chunks'
|
||||
chunks = split_video(str(input_path), output_dir, args.chunk_duration, args.verbose)
|
||||
print(f"\nCreated {len(chunks)} chunks in {output_dir}")
|
||||
sys.exit(0)
|
||||
|
||||
if not args.output:
|
||||
parser.error("--output required for single file processing")
|
||||
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Determine file type
|
||||
ext = input_path.suffix.lower()
|
||||
|
||||
if ext in ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.flv']:
|
||||
success = optimize_video(
|
||||
str(input_path),
|
||||
str(output_path),
|
||||
target_size_mb=args.target_size,
|
||||
quality=args.quality,
|
||||
resolution=args.resolution,
|
||||
verbose=args.verbose
|
||||
)
|
||||
elif ext in ['.mp3', '.wav', '.m4a', '.flac', '.aac']:
|
||||
success = optimize_audio(
|
||||
str(input_path),
|
||||
str(output_path),
|
||||
target_size_mb=args.target_size,
|
||||
bitrate=args.bitrate,
|
||||
verbose=args.verbose
|
||||
)
|
||||
elif ext in ['.jpg', '.jpeg', '.png', '.webp']:
|
||||
success = optimize_image(
|
||||
str(input_path),
|
||||
str(output_path),
|
||||
max_width=args.max_width,
|
||||
quality=args.quality,
|
||||
verbose=args.verbose
|
||||
)
|
||||
else:
|
||||
print(f"Error: Unsupported file type: {ext}")
|
||||
sys.exit(1)
|
||||
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
# Batch processing
|
||||
if args.input_dir:
|
||||
if not args.output_dir:
|
||||
parser.error("--output-dir required for batch processing")
|
||||
|
||||
input_dir = Path(args.input_dir)
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Find all media files
|
||||
patterns = ['*.mp4', '*.mov', '*.avi', '*.mkv', '*.webm',
|
||||
'*.mp3', '*.wav', '*.m4a', '*.flac',
|
||||
'*.jpg', '*.jpeg', '*.png', '*.webp']
|
||||
|
||||
files = []
|
||||
for pattern in patterns:
|
||||
files.extend(input_dir.glob(pattern))
|
||||
|
||||
if not files:
|
||||
print(f"No media files found in {input_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Found {len(files)} files to process")
|
||||
|
||||
success_count = 0
|
||||
for input_file in files:
|
||||
output_file = output_dir / input_file.name
|
||||
|
||||
ext = input_file.suffix.lower()
|
||||
success = False
|
||||
|
||||
if ext in ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.flv']:
|
||||
success = optimize_video(str(input_file), str(output_file),
|
||||
quality=args.quality, verbose=args.verbose)
|
||||
elif ext in ['.mp3', '.wav', '.m4a', '.flac', '.aac']:
|
||||
success = optimize_audio(str(input_file), str(output_file),
|
||||
bitrate=args.bitrate, verbose=args.verbose)
|
||||
elif ext in ['.jpg', '.jpeg', '.png', '.webp']:
|
||||
success = optimize_image(str(input_file), str(output_file),
|
||||
max_width=args.max_width, quality=args.quality,
|
||||
verbose=args.verbose)
|
||||
|
||||
if success:
|
||||
success_count += 1
|
||||
|
||||
print(f"\nProcessed: {success_count}/{len(files)} files")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,26 @@
|
||||
# AI Multimodal Skill Dependencies
|
||||
# Python 3.10+ required
|
||||
|
||||
# Google Gemini API
|
||||
google-genai>=0.1.0
|
||||
|
||||
# PDF processing
|
||||
pypdf>=4.0.0
|
||||
|
||||
# Document conversion
|
||||
python-docx>=1.0.0
|
||||
docx2pdf>=0.1.8 # Windows only, optional on Linux/macOS
|
||||
|
||||
# Markdown processing
|
||||
markdown>=3.5.0
|
||||
|
||||
# Image processing
|
||||
Pillow>=10.0.0
|
||||
|
||||
# Environment variable management
|
||||
python-dotenv>=1.0.0
|
||||
|
||||
# Testing dependencies (dev)
|
||||
pytest>=8.0.0
|
||||
pytest-cov>=4.1.0
|
||||
pytest-mock>=3.12.0
|
||||
Binary file not shown.
@@ -0,0 +1,20 @@
|
||||
# Core dependencies
|
||||
google-genai>=0.2.0
|
||||
python-dotenv>=1.0.0
|
||||
|
||||
# Image processing
|
||||
pillow>=10.0.0
|
||||
|
||||
# PDF processing
|
||||
pypdf>=3.0.0
|
||||
|
||||
# Document conversion
|
||||
markdown>=3.5
|
||||
|
||||
# Testing
|
||||
pytest>=7.4.0
|
||||
pytest-cov>=4.1.0
|
||||
pytest-mock>=3.12.0
|
||||
|
||||
# Optional dependencies for full functionality
|
||||
# ffmpeg-python>=0.2.0 # For media optimization (requires ffmpeg installed)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Tests for document_converter.py
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock, mock_open
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import document_converter as dc
|
||||
|
||||
|
||||
class TestAPIKeyFinder:
|
||||
"""Test API key finding logic."""
|
||||
|
||||
@patch.dict('os.environ', {'GEMINI_API_KEY': 'test-key-from-env'})
|
||||
def test_find_api_key_from_env(self):
|
||||
"""Test finding API key from environment."""
|
||||
api_key = dc.find_api_key()
|
||||
assert api_key == 'test-key-from-env'
|
||||
|
||||
@patch.dict('os.environ', {}, clear=True)
|
||||
@patch('document_converter.load_dotenv', None)
|
||||
def test_find_api_key_no_key(self):
|
||||
"""Test when no API key is available."""
|
||||
api_key = dc.find_api_key()
|
||||
assert api_key is None
|
||||
|
||||
|
||||
class TestProjectRoot:
|
||||
"""Test project root finding."""
|
||||
|
||||
@patch('pathlib.Path.exists')
|
||||
def test_find_project_root_with_git(self, mock_exists):
|
||||
"""Test finding project root with .git directory."""
|
||||
root = dc.find_project_root()
|
||||
assert isinstance(root, Path)
|
||||
|
||||
|
||||
class TestMimeType:
|
||||
"""Test MIME type detection."""
|
||||
|
||||
def test_pdf_mime_type(self):
|
||||
"""Test PDF MIME type."""
|
||||
assert dc.get_mime_type('document.pdf') == 'application/pdf'
|
||||
|
||||
def test_image_mime_types(self):
|
||||
"""Test image MIME types."""
|
||||
assert dc.get_mime_type('image.jpg') == 'image/jpeg'
|
||||
assert dc.get_mime_type('image.png') == 'image/png'
|
||||
|
||||
def test_unknown_mime_type(self):
|
||||
"""Test unknown file extension."""
|
||||
assert dc.get_mime_type('file.unknown') == 'application/octet-stream'
|
||||
|
||||
|
||||
class TestIntegration:
|
||||
"""Integration tests."""
|
||||
|
||||
def test_mime_type_integration(self):
|
||||
"""Test MIME type detection with various extensions."""
|
||||
test_cases = [
|
||||
('document.pdf', 'application/pdf'),
|
||||
('image.jpg', 'image/jpeg'),
|
||||
('unknown.xyz', 'application/octet-stream'),
|
||||
]
|
||||
for file_path, expected_mime in test_cases:
|
||||
assert dc.get_mime_type(file_path) == expected_mime
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v', '--cov=document_converter', '--cov-report=term-missing'])
|
||||
@@ -0,0 +1,362 @@
|
||||
"""
|
||||
Tests for gemini_batch_process.py
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import gemini_batch_process as gbp
|
||||
|
||||
|
||||
class TestAPIKeyFinder:
|
||||
"""Test API key detection."""
|
||||
|
||||
def test_find_api_key_from_env(self, monkeypatch):
|
||||
"""Test finding API key from environment variable."""
|
||||
monkeypatch.setenv('GEMINI_API_KEY', 'test_key_123')
|
||||
assert gbp.find_api_key() == 'test_key_123'
|
||||
|
||||
@patch('gemini_batch_process.load_dotenv')
|
||||
def test_find_api_key_not_found(self, mock_load_dotenv, monkeypatch):
|
||||
"""Test when API key is not found."""
|
||||
monkeypatch.delenv('GEMINI_API_KEY', raising=False)
|
||||
# Mock load_dotenv to not actually load any files
|
||||
mock_load_dotenv.return_value = None
|
||||
assert gbp.find_api_key() is None
|
||||
|
||||
|
||||
class TestMimeTypeDetection:
|
||||
"""Test MIME type detection."""
|
||||
|
||||
def test_audio_mime_types(self):
|
||||
"""Test audio file MIME types."""
|
||||
assert gbp.get_mime_type('test.mp3') == 'audio/mp3'
|
||||
assert gbp.get_mime_type('test.wav') == 'audio/wav'
|
||||
assert gbp.get_mime_type('test.aac') == 'audio/aac'
|
||||
assert gbp.get_mime_type('test.flac') == 'audio/flac'
|
||||
|
||||
def test_image_mime_types(self):
|
||||
"""Test image file MIME types."""
|
||||
assert gbp.get_mime_type('test.jpg') == 'image/jpeg'
|
||||
assert gbp.get_mime_type('test.jpeg') == 'image/jpeg'
|
||||
assert gbp.get_mime_type('test.png') == 'image/png'
|
||||
assert gbp.get_mime_type('test.webp') == 'image/webp'
|
||||
|
||||
def test_video_mime_types(self):
|
||||
"""Test video file MIME types."""
|
||||
assert gbp.get_mime_type('test.mp4') == 'video/mp4'
|
||||
assert gbp.get_mime_type('test.mov') == 'video/quicktime'
|
||||
assert gbp.get_mime_type('test.avi') == 'video/x-msvideo'
|
||||
|
||||
def test_document_mime_types(self):
|
||||
"""Test document file MIME types."""
|
||||
assert gbp.get_mime_type('test.pdf') == 'application/pdf'
|
||||
assert gbp.get_mime_type('test.txt') == 'text/plain'
|
||||
|
||||
def test_unknown_mime_type(self):
|
||||
"""Test unknown file extension."""
|
||||
assert gbp.get_mime_type('test.xyz') == 'application/octet-stream'
|
||||
|
||||
def test_case_insensitive(self):
|
||||
"""Test case-insensitive extension matching."""
|
||||
assert gbp.get_mime_type('TEST.MP3') == 'audio/mp3'
|
||||
assert gbp.get_mime_type('Test.JPG') == 'image/jpeg'
|
||||
|
||||
|
||||
class TestFileUpload:
|
||||
"""Test file upload functionality."""
|
||||
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
def test_upload_file_success(self, mock_client_class):
|
||||
"""Test successful file upload."""
|
||||
# Mock client and file
|
||||
mock_client = Mock()
|
||||
mock_file = Mock()
|
||||
mock_file.state.name = 'ACTIVE'
|
||||
mock_file.name = 'test_file'
|
||||
mock_client.files.upload.return_value = mock_file
|
||||
|
||||
result = gbp.upload_file(mock_client, 'test.jpg', verbose=False)
|
||||
|
||||
assert result == mock_file
|
||||
mock_client.files.upload.assert_called_once_with(file='test.jpg')
|
||||
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
@patch('gemini_batch_process.time.sleep')
|
||||
def test_upload_video_with_processing(self, mock_sleep, mock_client_class):
|
||||
"""Test video upload with processing wait."""
|
||||
mock_client = Mock()
|
||||
|
||||
# First call: PROCESSING, second call: ACTIVE
|
||||
mock_file_processing = Mock()
|
||||
mock_file_processing.state.name = 'PROCESSING'
|
||||
mock_file_processing.name = 'test_video'
|
||||
|
||||
mock_file_active = Mock()
|
||||
mock_file_active.state.name = 'ACTIVE'
|
||||
mock_file_active.name = 'test_video'
|
||||
|
||||
mock_client.files.upload.return_value = mock_file_processing
|
||||
mock_client.files.get.return_value = mock_file_active
|
||||
|
||||
result = gbp.upload_file(mock_client, 'test.mp4', verbose=False)
|
||||
|
||||
assert result.state.name == 'ACTIVE'
|
||||
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
def test_upload_file_failed(self, mock_client_class):
|
||||
"""Test failed file upload."""
|
||||
mock_client = Mock()
|
||||
mock_file = Mock()
|
||||
mock_file.state.name = 'FAILED'
|
||||
mock_client.files.upload.return_value = mock_file
|
||||
mock_client.files.get.return_value = mock_file
|
||||
|
||||
with pytest.raises(ValueError, match="File processing failed"):
|
||||
gbp.upload_file(mock_client, 'test.mp4', verbose=False)
|
||||
|
||||
|
||||
class TestProcessFile:
|
||||
"""Test file processing functionality."""
|
||||
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
@patch('builtins.open', create=True)
|
||||
@patch('pathlib.Path.stat')
|
||||
def test_process_small_file_inline(self, mock_stat, mock_open, mock_client_class):
|
||||
"""Test processing small file with inline data."""
|
||||
# Mock small file
|
||||
mock_stat.return_value.st_size = 10 * 1024 * 1024 # 10MB
|
||||
|
||||
# Mock file content
|
||||
mock_open.return_value.__enter__.return_value.read.return_value = b'test_data'
|
||||
|
||||
# Mock client and response
|
||||
mock_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.text = 'Test response'
|
||||
mock_client.models.generate_content.return_value = mock_response
|
||||
|
||||
result = gbp.process_file(
|
||||
client=mock_client,
|
||||
file_path='test.jpg',
|
||||
prompt='Describe this image',
|
||||
model='gemini-2.5-flash',
|
||||
task='analyze',
|
||||
format_output='text',
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result['status'] == 'success'
|
||||
assert result['response'] == 'Test response'
|
||||
|
||||
@patch('gemini_batch_process.upload_file')
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
@patch('pathlib.Path.stat')
|
||||
def test_process_large_file_api(self, mock_stat, mock_client_class, mock_upload):
|
||||
"""Test processing large file with File API."""
|
||||
# Mock large file
|
||||
mock_stat.return_value.st_size = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
# Mock upload and response
|
||||
mock_file = Mock()
|
||||
mock_upload.return_value = mock_file
|
||||
|
||||
mock_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.text = 'Test response'
|
||||
mock_client.models.generate_content.return_value = mock_response
|
||||
|
||||
result = gbp.process_file(
|
||||
client=mock_client,
|
||||
file_path='test.mp4',
|
||||
prompt='Summarize this video',
|
||||
model='gemini-2.5-flash',
|
||||
task='analyze',
|
||||
format_output='text',
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result['status'] == 'success'
|
||||
mock_upload.assert_called_once()
|
||||
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
@patch('builtins.open', create=True)
|
||||
@patch('pathlib.Path.stat')
|
||||
def test_process_file_error_handling(self, mock_stat, mock_open, mock_client_class):
|
||||
"""Test error handling in file processing."""
|
||||
mock_stat.return_value.st_size = 1024
|
||||
|
||||
# Mock file read
|
||||
mock_file = MagicMock()
|
||||
mock_file.__enter__.return_value.read.return_value = b'test_data'
|
||||
mock_open.return_value = mock_file
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.models.generate_content.side_effect = Exception("API Error")
|
||||
|
||||
result = gbp.process_file(
|
||||
client=mock_client,
|
||||
file_path='test.jpg',
|
||||
prompt='Test',
|
||||
model='gemini-2.5-flash',
|
||||
task='analyze',
|
||||
format_output='text',
|
||||
verbose=False,
|
||||
max_retries=1
|
||||
)
|
||||
|
||||
assert result['status'] == 'error'
|
||||
assert 'API Error' in result['error']
|
||||
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
@patch('builtins.open', create=True)
|
||||
@patch('pathlib.Path.stat')
|
||||
def test_image_generation_with_aspect_ratio(self, mock_stat, mock_open, mock_client_class):
|
||||
"""Test image generation with aspect ratio config."""
|
||||
mock_stat.return_value.st_size = 1024
|
||||
|
||||
# Mock file read
|
||||
mock_file = MagicMock()
|
||||
mock_file.__enter__.return_value.read.return_value = b'test'
|
||||
mock_open.return_value = mock_file
|
||||
|
||||
mock_client = Mock()
|
||||
mock_response = Mock()
|
||||
mock_response.candidates = [Mock()]
|
||||
mock_response.candidates[0].content.parts = [
|
||||
Mock(inline_data=Mock(data=b'fake_image_data'))
|
||||
]
|
||||
mock_client.models.generate_content.return_value = mock_response
|
||||
|
||||
result = gbp.process_file(
|
||||
client=mock_client,
|
||||
file_path='test.txt',
|
||||
prompt='Generate mountain landscape',
|
||||
model='gemini-2.5-flash-image',
|
||||
task='generate',
|
||||
format_output='text',
|
||||
aspect_ratio='16:9',
|
||||
verbose=False
|
||||
)
|
||||
|
||||
# Verify config was called with correct structure
|
||||
call_args = mock_client.models.generate_content.call_args
|
||||
config = call_args.kwargs.get('config')
|
||||
assert config is not None
|
||||
assert result['status'] == 'success'
|
||||
assert 'generated_image' in result
|
||||
|
||||
|
||||
class TestBatchProcessing:
|
||||
"""Test batch processing functionality."""
|
||||
|
||||
@patch('gemini_batch_process.find_api_key')
|
||||
@patch('gemini_batch_process.process_file')
|
||||
@patch('gemini_batch_process.genai.Client')
|
||||
def test_batch_process_success(self, mock_client_class, mock_process, mock_find_key):
|
||||
"""Test successful batch processing."""
|
||||
mock_find_key.return_value = 'test_key'
|
||||
mock_process.return_value = {'status': 'success', 'response': 'Test'}
|
||||
|
||||
results = gbp.batch_process(
|
||||
files=['test1.jpg', 'test2.jpg'],
|
||||
prompt='Analyze',
|
||||
model='gemini-2.5-flash',
|
||||
task='analyze',
|
||||
format_output='text',
|
||||
verbose=False,
|
||||
dry_run=False
|
||||
)
|
||||
|
||||
assert len(results) == 2
|
||||
assert all(r['status'] == 'success' for r in results)
|
||||
|
||||
@patch('gemini_batch_process.find_api_key')
|
||||
def test_batch_process_no_api_key(self, mock_find_key):
|
||||
"""Test batch processing without API key."""
|
||||
mock_find_key.return_value = None
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
gbp.batch_process(
|
||||
files=['test.jpg'],
|
||||
prompt='Test',
|
||||
model='gemini-2.5-flash',
|
||||
task='analyze',
|
||||
format_output='text',
|
||||
verbose=False,
|
||||
dry_run=False
|
||||
)
|
||||
|
||||
@patch('gemini_batch_process.find_api_key')
|
||||
def test_batch_process_dry_run(self, mock_find_key):
|
||||
"""Test dry run mode."""
|
||||
# API key not needed for dry run, but we mock it to avoid sys.exit
|
||||
mock_find_key.return_value = 'test_key'
|
||||
|
||||
results = gbp.batch_process(
|
||||
files=['test1.jpg', 'test2.jpg'],
|
||||
prompt='Test',
|
||||
model='gemini-2.5-flash',
|
||||
task='analyze',
|
||||
format_output='text',
|
||||
verbose=False,
|
||||
dry_run=True
|
||||
)
|
||||
|
||||
assert results == []
|
||||
|
||||
|
||||
class TestResultsSaving:
|
||||
"""Test results saving functionality."""
|
||||
|
||||
@patch('builtins.open', create=True)
|
||||
@patch('json.dump')
|
||||
def test_save_results_json(self, mock_json_dump, mock_open):
|
||||
"""Test saving results as JSON."""
|
||||
results = [
|
||||
{'file': 'test1.jpg', 'status': 'success', 'response': 'Test1'},
|
||||
{'file': 'test2.jpg', 'status': 'success', 'response': 'Test2'}
|
||||
]
|
||||
|
||||
gbp.save_results(results, 'output.json', 'json')
|
||||
|
||||
mock_json_dump.assert_called_once()
|
||||
|
||||
@patch('builtins.open', create=True)
|
||||
@patch('csv.DictWriter')
|
||||
def test_save_results_csv(self, mock_csv_writer, mock_open):
|
||||
"""Test saving results as CSV."""
|
||||
results = [
|
||||
{'file': 'test1.jpg', 'status': 'success', 'response': 'Test1'},
|
||||
{'file': 'test2.jpg', 'status': 'success', 'response': 'Test2'}
|
||||
]
|
||||
|
||||
gbp.save_results(results, 'output.csv', 'csv')
|
||||
|
||||
# Verify CSV writer was used
|
||||
mock_csv_writer.assert_called_once()
|
||||
|
||||
@patch('builtins.open', create=True)
|
||||
def test_save_results_markdown(self, mock_open):
|
||||
"""Test saving results as Markdown."""
|
||||
mock_file = MagicMock()
|
||||
mock_open.return_value.__enter__.return_value = mock_file
|
||||
|
||||
results = [
|
||||
{'file': 'test1.jpg', 'status': 'success', 'response': 'Test1'},
|
||||
{'file': 'test2.jpg', 'status': 'error', 'error': 'Failed'}
|
||||
]
|
||||
|
||||
gbp.save_results(results, 'output.md', 'markdown')
|
||||
|
||||
# Verify write was called
|
||||
assert mock_file.write.call_count > 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v', '--cov=gemini_batch_process', '--cov-report=term-missing'])
|
||||
@@ -0,0 +1,373 @@
|
||||
"""
|
||||
Tests for media_optimizer.py
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
import json
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import media_optimizer as mo
|
||||
|
||||
|
||||
class TestEnvLoading:
|
||||
"""Test environment variable loading."""
|
||||
|
||||
@patch('media_optimizer.load_dotenv')
|
||||
@patch('pathlib.Path.exists')
|
||||
def test_load_env_files_success(self, mock_exists, mock_load_dotenv):
|
||||
"""Test successful .env file loading."""
|
||||
mock_exists.return_value = True
|
||||
mo.load_env_files()
|
||||
# Should be called for skill, skills, and claude dirs
|
||||
assert mock_load_dotenv.call_count >= 1
|
||||
|
||||
@patch('media_optimizer.load_dotenv', None)
|
||||
def test_load_env_files_no_dotenv(self):
|
||||
"""Test when dotenv is not available."""
|
||||
# Should not raise an error
|
||||
mo.load_env_files()
|
||||
|
||||
|
||||
class TestFFmpegCheck:
|
||||
"""Test ffmpeg availability checking."""
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_ffmpeg_installed(self, mock_run):
|
||||
"""Test when ffmpeg is installed."""
|
||||
mock_run.return_value = Mock()
|
||||
assert mo.check_ffmpeg() is True
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_ffmpeg_not_installed(self, mock_run):
|
||||
"""Test when ffmpeg is not installed."""
|
||||
mock_run.side_effect = FileNotFoundError()
|
||||
assert mo.check_ffmpeg() is False
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_ffmpeg_error(self, mock_run):
|
||||
"""Test ffmpeg command error."""
|
||||
mock_run.side_effect = Exception("Error")
|
||||
assert mo.check_ffmpeg() is False
|
||||
|
||||
|
||||
class TestMediaInfo:
|
||||
"""Test media information extraction."""
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('subprocess.run')
|
||||
def test_get_video_info(self, mock_run, mock_check):
|
||||
"""Test extracting video information."""
|
||||
mock_check.return_value = True
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.stdout = json.dumps({
|
||||
'format': {
|
||||
'size': '10485760',
|
||||
'duration': '120.5',
|
||||
'bit_rate': '691200'
|
||||
},
|
||||
'streams': [
|
||||
{
|
||||
'codec_type': 'video',
|
||||
'width': 1920,
|
||||
'height': 1080,
|
||||
'r_frame_rate': '30/1'
|
||||
},
|
||||
{
|
||||
'codec_type': 'audio',
|
||||
'sample_rate': '48000',
|
||||
'channels': 2
|
||||
}
|
||||
]
|
||||
})
|
||||
mock_run.return_value = mock_result
|
||||
|
||||
info = mo.get_media_info('test.mp4')
|
||||
|
||||
assert info['size'] == 10485760
|
||||
assert info['duration'] == 120.5
|
||||
assert info['width'] == 1920
|
||||
assert info['height'] == 1080
|
||||
assert info['sample_rate'] == 48000
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
def test_get_media_info_no_ffmpeg(self, mock_check):
|
||||
"""Test when ffmpeg is not available."""
|
||||
mock_check.return_value = False
|
||||
info = mo.get_media_info('test.mp4')
|
||||
assert info == {}
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('subprocess.run')
|
||||
def test_get_media_info_error(self, mock_run, mock_check):
|
||||
"""Test error handling in media info extraction."""
|
||||
mock_check.return_value = True
|
||||
mock_run.side_effect = Exception("Error")
|
||||
|
||||
info = mo.get_media_info('test.mp4')
|
||||
assert info == {}
|
||||
|
||||
|
||||
class TestVideoOptimization:
|
||||
"""Test video optimization functionality."""
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('media_optimizer.get_media_info')
|
||||
@patch('subprocess.run')
|
||||
def test_optimize_video_success(self, mock_run, mock_info, mock_check):
|
||||
"""Test successful video optimization."""
|
||||
mock_check.return_value = True
|
||||
mock_info.side_effect = [
|
||||
# Input info
|
||||
{
|
||||
'size': 50 * 1024 * 1024,
|
||||
'duration': 120.0,
|
||||
'bit_rate': 3500000,
|
||||
'width': 1920,
|
||||
'height': 1080
|
||||
},
|
||||
# Output info
|
||||
{
|
||||
'size': 25 * 1024 * 1024,
|
||||
'duration': 120.0,
|
||||
'width': 1920,
|
||||
'height': 1080
|
||||
}
|
||||
]
|
||||
|
||||
result = mo.optimize_video(
|
||||
'input.mp4',
|
||||
'output.mp4',
|
||||
quality=23,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result is True
|
||||
mock_run.assert_called_once()
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
def test_optimize_video_no_ffmpeg(self, mock_check):
|
||||
"""Test video optimization without ffmpeg."""
|
||||
mock_check.return_value = False
|
||||
|
||||
result = mo.optimize_video('input.mp4', 'output.mp4')
|
||||
assert result is False
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('media_optimizer.get_media_info')
|
||||
def test_optimize_video_no_info(self, mock_info, mock_check):
|
||||
"""Test video optimization when info cannot be read."""
|
||||
mock_check.return_value = True
|
||||
mock_info.return_value = {}
|
||||
|
||||
result = mo.optimize_video('input.mp4', 'output.mp4')
|
||||
assert result is False
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('media_optimizer.get_media_info')
|
||||
@patch('subprocess.run')
|
||||
def test_optimize_video_with_target_size(self, mock_run, mock_info, mock_check):
|
||||
"""Test video optimization with target size."""
|
||||
mock_check.return_value = True
|
||||
mock_info.side_effect = [
|
||||
{'size': 100 * 1024 * 1024, 'duration': 60.0, 'bit_rate': 3500000},
|
||||
{'size': 50 * 1024 * 1024, 'duration': 60.0}
|
||||
]
|
||||
|
||||
result = mo.optimize_video(
|
||||
'input.mp4',
|
||||
'output.mp4',
|
||||
target_size_mb=50,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('media_optimizer.get_media_info')
|
||||
@patch('subprocess.run')
|
||||
def test_optimize_video_with_resolution(self, mock_run, mock_info, mock_check):
|
||||
"""Test video optimization with custom resolution."""
|
||||
mock_check.return_value = True
|
||||
mock_info.side_effect = [
|
||||
{'size': 50 * 1024 * 1024, 'duration': 120.0, 'bit_rate': 3500000},
|
||||
{'size': 25 * 1024 * 1024, 'duration': 120.0}
|
||||
]
|
||||
|
||||
result = mo.optimize_video(
|
||||
'input.mp4',
|
||||
'output.mp4',
|
||||
resolution='1280x720',
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
class TestAudioOptimization:
|
||||
"""Test audio optimization functionality."""
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('media_optimizer.get_media_info')
|
||||
@patch('subprocess.run')
|
||||
def test_optimize_audio_success(self, mock_run, mock_info, mock_check):
|
||||
"""Test successful audio optimization."""
|
||||
mock_check.return_value = True
|
||||
mock_info.side_effect = [
|
||||
{'size': 10 * 1024 * 1024, 'duration': 300.0},
|
||||
{'size': 5 * 1024 * 1024, 'duration': 300.0}
|
||||
]
|
||||
|
||||
result = mo.optimize_audio(
|
||||
'input.mp3',
|
||||
'output.m4a',
|
||||
bitrate='64k',
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result is True
|
||||
mock_run.assert_called_once()
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
def test_optimize_audio_no_ffmpeg(self, mock_check):
|
||||
"""Test audio optimization without ffmpeg."""
|
||||
mock_check.return_value = False
|
||||
|
||||
result = mo.optimize_audio('input.mp3', 'output.m4a')
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestImageOptimization:
|
||||
"""Test image optimization functionality."""
|
||||
|
||||
@patch('PIL.Image.open')
|
||||
@patch('pathlib.Path.stat')
|
||||
def test_optimize_image_success(self, mock_stat, mock_image_open):
|
||||
"""Test successful image optimization."""
|
||||
# Mock image
|
||||
mock_resized = Mock()
|
||||
mock_resized.mode = 'RGB'
|
||||
|
||||
mock_img = Mock()
|
||||
mock_img.width = 3840
|
||||
mock_img.height = 2160
|
||||
mock_img.mode = 'RGB'
|
||||
mock_img.resize.return_value = mock_resized
|
||||
mock_image_open.return_value = mock_img
|
||||
|
||||
# Mock file sizes
|
||||
mock_stat.return_value.st_size = 5 * 1024 * 1024
|
||||
|
||||
result = mo.optimize_image(
|
||||
'input.jpg',
|
||||
'output.jpg',
|
||||
max_width=1920,
|
||||
quality=85,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result is True
|
||||
# Since image is resized, save is called on the resized image
|
||||
mock_resized.save.assert_called_once()
|
||||
|
||||
@patch('PIL.Image.open')
|
||||
@patch('pathlib.Path.stat')
|
||||
def test_optimize_image_resize(self, mock_stat, mock_image_open):
|
||||
"""Test image resizing during optimization."""
|
||||
mock_img = Mock()
|
||||
mock_img.width = 3840
|
||||
mock_img.height = 2160
|
||||
mock_img.mode = 'RGB'
|
||||
mock_resized = Mock()
|
||||
mock_img.resize.return_value = mock_resized
|
||||
mock_image_open.return_value = mock_img
|
||||
|
||||
mock_stat.return_value.st_size = 5 * 1024 * 1024
|
||||
|
||||
mo.optimize_image('input.jpg', 'output.jpg', max_width=1920, verbose=False)
|
||||
|
||||
mock_img.resize.assert_called_once()
|
||||
|
||||
@patch('PIL.Image.open')
|
||||
@patch('pathlib.Path.stat')
|
||||
def test_optimize_image_rgba_to_jpg(self, mock_stat, mock_image_open):
|
||||
"""Test converting RGBA to RGB for JPEG."""
|
||||
mock_img = Mock()
|
||||
mock_img.width = 1920
|
||||
mock_img.height = 1080
|
||||
mock_img.mode = 'RGBA'
|
||||
mock_img.split.return_value = [Mock(), Mock(), Mock(), Mock()]
|
||||
mock_image_open.return_value = mock_img
|
||||
|
||||
mock_stat.return_value.st_size = 1024 * 1024
|
||||
|
||||
with patch('PIL.Image.new') as mock_new:
|
||||
mock_rgb = Mock()
|
||||
mock_new.return_value = mock_rgb
|
||||
|
||||
mo.optimize_image('input.png', 'output.jpg', verbose=False)
|
||||
|
||||
mock_new.assert_called_once()
|
||||
|
||||
def test_optimize_image_no_pillow(self):
|
||||
"""Test image optimization without Pillow."""
|
||||
with patch.dict('sys.modules', {'PIL': None}):
|
||||
result = mo.optimize_image('input.jpg', 'output.jpg')
|
||||
# Will fail to import but function handles it
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestVideoSplitting:
|
||||
"""Test video splitting functionality."""
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('media_optimizer.get_media_info')
|
||||
@patch('subprocess.run')
|
||||
@patch('pathlib.Path.mkdir')
|
||||
def test_split_video_success(self, mock_mkdir, mock_run, mock_info, mock_check):
|
||||
"""Test successful video splitting."""
|
||||
mock_check.return_value = True
|
||||
mock_info.return_value = {'duration': 7200.0} # 2 hours
|
||||
|
||||
result = mo.split_video(
|
||||
'input.mp4',
|
||||
'./chunks',
|
||||
chunk_duration=3600, # 1 hour chunks
|
||||
verbose=False
|
||||
)
|
||||
|
||||
# Duration 7200s / 3600s = 2, +1 for safety = 3 chunks
|
||||
assert len(result) == 3
|
||||
assert mock_run.call_count == 3
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
@patch('media_optimizer.get_media_info')
|
||||
def test_split_video_short_duration(self, mock_info, mock_check):
|
||||
"""Test splitting video shorter than chunk duration."""
|
||||
mock_check.return_value = True
|
||||
mock_info.return_value = {'duration': 1800.0} # 30 minutes
|
||||
|
||||
result = mo.split_video(
|
||||
'input.mp4',
|
||||
'./chunks',
|
||||
chunk_duration=3600, # 1 hour
|
||||
verbose=False
|
||||
)
|
||||
|
||||
assert result == ['input.mp4']
|
||||
|
||||
@patch('media_optimizer.check_ffmpeg')
|
||||
def test_split_video_no_ffmpeg(self, mock_check):
|
||||
"""Test video splitting without ffmpeg."""
|
||||
mock_check.return_value = False
|
||||
|
||||
result = mo.split_video('input.mp4', './chunks')
|
||||
assert result == []
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v', '--cov=media_optimizer', '--cov-report=term-missing'])
|
||||
@@ -24,6 +24,7 @@
|
||||
"framer-motion": "^12.34.3",
|
||||
"highlight.js": "^11.11.1",
|
||||
"i18next": "^25.8.15",
|
||||
"jszip": "^3.10.1",
|
||||
"lucide-react": "^0.468.0",
|
||||
"radix-ui": "^1.4.3",
|
||||
"react": "^19.0.0",
|
||||
|
||||
Generated
+80
@@ -50,6 +50,9 @@ importers:
|
||||
i18next:
|
||||
specifier: ^25.8.15
|
||||
version: 25.8.15(typescript@5.7.3)
|
||||
jszip:
|
||||
specifier: ^3.10.1
|
||||
version: 3.10.1
|
||||
lucide-react:
|
||||
specifier: ^0.468.0
|
||||
version: 0.468.0(react@19.2.4)
|
||||
@@ -1550,6 +1553,9 @@ packages:
|
||||
resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
core-util-is@1.0.3:
|
||||
resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
|
||||
|
||||
cssesc@3.0.0:
|
||||
resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
|
||||
engines: {node: '>=4'}
|
||||
@@ -1765,12 +1771,18 @@ packages:
|
||||
typescript:
|
||||
optional: true
|
||||
|
||||
immediate@3.0.6:
|
||||
resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}
|
||||
|
||||
immer@10.2.0:
|
||||
resolution: {integrity: sha512-d/+XTN3zfODyjr89gM3mPq1WNX2B8pYsu7eORitdwyA2sBubnTl3laYlBk4sXY5FUa5qTZGBDPJICVbvqzjlbw==}
|
||||
|
||||
immer@11.1.4:
|
||||
resolution: {integrity: sha512-XREFCPo6ksxVzP4E0ekD5aMdf8WMwmdNaz6vuvxgI40UaEiu6q3p8X52aU6GdyvLY3XXX/8R7JOTXStz/nBbRw==}
|
||||
|
||||
inherits@2.0.4:
|
||||
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
||||
|
||||
inline-style-parser@0.2.7:
|
||||
resolution: {integrity: sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==}
|
||||
|
||||
@@ -1794,6 +1806,9 @@ packages:
|
||||
resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
isarray@1.0.0:
|
||||
resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==}
|
||||
|
||||
jiti@2.6.1:
|
||||
resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==}
|
||||
hasBin: true
|
||||
@@ -1811,6 +1826,12 @@ packages:
|
||||
engines: {node: '>=6'}
|
||||
hasBin: true
|
||||
|
||||
jszip@3.10.1:
|
||||
resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==}
|
||||
|
||||
lie@3.3.0:
|
||||
resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==}
|
||||
|
||||
lightningcss-android-arm64@1.31.1:
|
||||
resolution: {integrity: sha512-HXJF3x8w9nQ4jbXRiNppBCqeZPIAfUo8zE/kOEGbW5NZvGc/K7nMxbhIr+YlFlHW5mpbg/YFPdbnCh1wAXCKFg==}
|
||||
engines: {node: '>= 12.0.0'}
|
||||
@@ -2051,6 +2072,9 @@ packages:
|
||||
node-releases@2.0.27:
|
||||
resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==}
|
||||
|
||||
pako@1.0.11:
|
||||
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
|
||||
|
||||
parse-entities@4.0.2:
|
||||
resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==}
|
||||
|
||||
@@ -2069,6 +2093,9 @@ packages:
|
||||
resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==}
|
||||
engines: {node: ^10 || ^12 || >=14}
|
||||
|
||||
process-nextick-args@2.0.1:
|
||||
resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}
|
||||
|
||||
property-information@7.1.0:
|
||||
resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==}
|
||||
|
||||
@@ -2175,6 +2202,9 @@ packages:
|
||||
resolution: {integrity: sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
readable-stream@2.3.8:
|
||||
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
|
||||
|
||||
recharts@3.8.0:
|
||||
resolution: {integrity: sha512-Z/m38DX3L73ExO4Tpc9/iZWHmHnlzWG4njQbxsF5aSjwqmHNDDIm0rdEBArkwsBvR8U6EirlEHiQNYWCVh9sGQ==}
|
||||
engines: {node: '>=18'}
|
||||
@@ -2214,6 +2244,9 @@ packages:
|
||||
engines: {node: '>=18.0.0', npm: '>=8.0.0'}
|
||||
hasBin: true
|
||||
|
||||
safe-buffer@5.1.2:
|
||||
resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==}
|
||||
|
||||
scheduler@0.27.0:
|
||||
resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==}
|
||||
|
||||
@@ -2224,6 +2257,9 @@ packages:
|
||||
set-cookie-parser@2.7.2:
|
||||
resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==}
|
||||
|
||||
setimmediate@1.0.5:
|
||||
resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==}
|
||||
|
||||
source-map-js@1.2.1:
|
||||
resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
@@ -2231,6 +2267,9 @@ packages:
|
||||
space-separated-tokens@2.0.2:
|
||||
resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==}
|
||||
|
||||
string_decoder@1.1.1:
|
||||
resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==}
|
||||
|
||||
stringify-entities@4.0.4:
|
||||
resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==}
|
||||
|
||||
@@ -3799,6 +3838,8 @@ snapshots:
|
||||
|
||||
cookie@1.1.1: {}
|
||||
|
||||
core-util-is@1.0.3: {}
|
||||
|
||||
cssesc@3.0.0: {}
|
||||
|
||||
csstype@3.2.3: {}
|
||||
@@ -4016,10 +4057,14 @@ snapshots:
|
||||
optionalDependencies:
|
||||
typescript: 5.7.3
|
||||
|
||||
immediate@3.0.6: {}
|
||||
|
||||
immer@10.2.0: {}
|
||||
|
||||
immer@11.1.4: {}
|
||||
|
||||
inherits@2.0.4: {}
|
||||
|
||||
inline-style-parser@0.2.7: {}
|
||||
|
||||
internmap@2.0.3: {}
|
||||
@@ -4037,6 +4082,8 @@ snapshots:
|
||||
|
||||
is-plain-obj@4.1.0: {}
|
||||
|
||||
isarray@1.0.0: {}
|
||||
|
||||
jiti@2.6.1: {}
|
||||
|
||||
js-tokens@4.0.0: {}
|
||||
@@ -4045,6 +4092,17 @@ snapshots:
|
||||
|
||||
json5@2.2.3: {}
|
||||
|
||||
jszip@3.10.1:
|
||||
dependencies:
|
||||
lie: 3.3.0
|
||||
pako: 1.0.11
|
||||
readable-stream: 2.3.8
|
||||
setimmediate: 1.0.5
|
||||
|
||||
lie@3.3.0:
|
||||
dependencies:
|
||||
immediate: 3.0.6
|
||||
|
||||
lightningcss-android-arm64@1.31.1:
|
||||
optional: true
|
||||
|
||||
@@ -4472,6 +4530,8 @@ snapshots:
|
||||
|
||||
node-releases@2.0.27: {}
|
||||
|
||||
pako@1.0.11: {}
|
||||
|
||||
parse-entities@4.0.2:
|
||||
dependencies:
|
||||
'@types/unist': 2.0.11
|
||||
@@ -4497,6 +4557,8 @@ snapshots:
|
||||
picocolors: 1.1.1
|
||||
source-map-js: 1.2.1
|
||||
|
||||
process-nextick-args@2.0.1: {}
|
||||
|
||||
property-information@7.1.0: {}
|
||||
|
||||
radix-ui@1.4.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4):
|
||||
@@ -4646,6 +4708,16 @@ snapshots:
|
||||
|
||||
react@19.2.4: {}
|
||||
|
||||
readable-stream@2.3.8:
|
||||
dependencies:
|
||||
core-util-is: 1.0.3
|
||||
inherits: 2.0.4
|
||||
isarray: 1.0.0
|
||||
process-nextick-args: 2.0.1
|
||||
safe-buffer: 5.1.2
|
||||
string_decoder: 1.1.1
|
||||
util-deprecate: 1.0.2
|
||||
|
||||
recharts@3.8.0(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react-is@19.2.4)(react@19.2.4)(redux@5.0.1):
|
||||
dependencies:
|
||||
'@reduxjs/toolkit': 2.11.2(react-redux@9.2.0(@types/react@19.2.14)(react@19.2.4)(redux@5.0.1))(react@19.2.4)
|
||||
@@ -4747,16 +4819,24 @@ snapshots:
|
||||
'@rollup/rollup-win32-x64-msvc': 4.57.1
|
||||
fsevents: 2.3.3
|
||||
|
||||
safe-buffer@5.1.2: {}
|
||||
|
||||
scheduler@0.27.0: {}
|
||||
|
||||
semver@6.3.1: {}
|
||||
|
||||
set-cookie-parser@2.7.2: {}
|
||||
|
||||
setimmediate@1.0.5: {}
|
||||
|
||||
source-map-js@1.2.1: {}
|
||||
|
||||
space-separated-tokens@2.0.2: {}
|
||||
|
||||
string_decoder@1.1.1:
|
||||
dependencies:
|
||||
safe-buffer: 5.1.2
|
||||
|
||||
stringify-entities@4.0.4:
|
||||
dependencies:
|
||||
character-entities-html4: 2.1.0
|
||||
|
||||
@@ -28,6 +28,10 @@ export function AgentSelector({ value, onChange }: AgentSelectorProps) {
|
||||
.then((res) => {
|
||||
const active = (res.agents ?? []).filter((a) => a.status === "active");
|
||||
setAgents(active);
|
||||
// Auto-select first agent if current value doesn't match any active agent
|
||||
if (active.length > 0 && !active.some((a) => a.agent_key === value)) {
|
||||
onChange(active[0]!.agent_key);
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
}, [http, connected]);
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
import { useState, useRef, useCallback, type KeyboardEvent } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Send, Square } from "lucide-react";
|
||||
import { Send, Square, Paperclip, X } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
export interface AttachedFile {
|
||||
file: File;
|
||||
/** Server path after upload, set during send */
|
||||
serverPath?: string;
|
||||
}
|
||||
|
||||
interface ChatInputProps {
|
||||
onSend: (message: string) => void;
|
||||
onSend: (message: string, files?: AttachedFile[]) => void;
|
||||
onAbort: () => void;
|
||||
isRunning: boolean;
|
||||
disabled?: boolean;
|
||||
@@ -13,17 +19,19 @@ interface ChatInputProps {
|
||||
export function ChatInput({ onSend, onAbort, isRunning, disabled }: ChatInputProps) {
|
||||
const { t } = useTranslation("common");
|
||||
const [value, setValue] = useState("");
|
||||
const [files, setFiles] = useState<AttachedFile[]>([]);
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const handleSend = useCallback(() => {
|
||||
if (!value.trim() || disabled) return;
|
||||
onSend(value);
|
||||
if ((!value.trim() && files.length === 0) || disabled) return;
|
||||
onSend(value, files.length > 0 ? files : undefined);
|
||||
setValue("");
|
||||
// Reset textarea height
|
||||
setFiles([]);
|
||||
if (textareaRef.current) {
|
||||
textareaRef.current.style.height = "auto";
|
||||
}
|
||||
}, [value, onSend, disabled]);
|
||||
}, [value, files, onSend, disabled]);
|
||||
|
||||
const handleKeyDown = useCallback(
|
||||
(e: KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
@@ -43,41 +51,101 @@ export function ChatInput({ onSend, onAbort, isRunning, disabled }: ChatInputPro
|
||||
el.style.height = Math.min(el.scrollHeight, 200) + "px";
|
||||
}, []);
|
||||
|
||||
const handleFileSelect = useCallback(() => {
|
||||
fileInputRef.current?.click();
|
||||
}, []);
|
||||
|
||||
const handleFileChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const selected = e.target.files;
|
||||
if (!selected) return;
|
||||
const newFiles: AttachedFile[] = Array.from(selected).map((f) => ({ file: f }));
|
||||
setFiles((prev) => [...prev, ...newFiles]);
|
||||
// Reset input so the same file can be re-selected
|
||||
e.target.value = "";
|
||||
}, []);
|
||||
|
||||
const removeFile = useCallback((index: number) => {
|
||||
setFiles((prev) => prev.filter((_, i) => i !== index));
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div
|
||||
className="flex items-end gap-2 border-t bg-background p-4 safe-bottom"
|
||||
style={{ paddingBottom: `calc(1rem + env(safe-area-inset-bottom) + var(--keyboard-height, 0px))` }}
|
||||
className="border-t bg-background safe-bottom"
|
||||
style={{ paddingBottom: `calc(env(safe-area-inset-bottom) + var(--keyboard-height, 0px))` }}
|
||||
>
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
value={value}
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
onInput={handleInput}
|
||||
placeholder={t("sendMessage")}
|
||||
disabled={disabled}
|
||||
rows={1}
|
||||
className="flex-1 resize-none rounded-lg border bg-background px-4 py-2.5 text-base md:text-sm placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-ring disabled:opacity-50"
|
||||
/>
|
||||
{isRunning ? (
|
||||
<Button
|
||||
variant="destructive"
|
||||
size="icon-lg"
|
||||
onClick={onAbort}
|
||||
title={t("stopGeneration")}
|
||||
>
|
||||
<Square className="h-4 w-4" />
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
size="icon-lg"
|
||||
onClick={handleSend}
|
||||
disabled={!value.trim() || disabled}
|
||||
title={t("sendMessageTitle")}
|
||||
>
|
||||
<Send className="h-4 w-4" />
|
||||
</Button>
|
||||
{/* Attached files preview */}
|
||||
{files.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 px-4 pt-3">
|
||||
{files.map((af, i) => (
|
||||
<span
|
||||
key={i}
|
||||
className="inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs"
|
||||
>
|
||||
<span className="max-w-[150px] truncate">{af.file.name}</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeFile(i)}
|
||||
className="rounded-sm p-0.5 hover:bg-accent"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-end gap-2 p-4 pt-3">
|
||||
{/* File attach button */}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-lg"
|
||||
onClick={handleFileSelect}
|
||||
disabled={disabled || isRunning}
|
||||
title={t("attachFile")}
|
||||
className="text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<Paperclip className="h-4 w-4" />
|
||||
</Button>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
multiple
|
||||
onChange={handleFileChange}
|
||||
className="hidden"
|
||||
/>
|
||||
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
value={value}
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
onInput={handleInput}
|
||||
placeholder={t("sendMessage")}
|
||||
disabled={disabled}
|
||||
rows={1}
|
||||
className="flex-1 resize-none rounded-lg border bg-background px-4 py-2.5 text-base md:text-sm placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-ring disabled:opacity-50"
|
||||
/>
|
||||
{isRunning ? (
|
||||
<Button
|
||||
variant="destructive"
|
||||
size="icon-lg"
|
||||
onClick={onAbort}
|
||||
title={t("stopGeneration")}
|
||||
>
|
||||
<Square className="h-4 w-4" />
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
size="icon-lg"
|
||||
onClick={handleSend}
|
||||
disabled={(!value.trim() && files.length === 0) || disabled}
|
||||
title={t("sendMessageTitle")}
|
||||
>
|
||||
<Send className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -76,6 +76,7 @@
|
||||
"sendMessage": "Send a message... (Shift+Enter for new line)",
|
||||
"stopGeneration": "Stop generation",
|
||||
"sendMessageTitle": "Send message",
|
||||
"attachFile": "Attach file",
|
||||
"thinking": "Thinking",
|
||||
"thinkingStreaming": "Thinking...",
|
||||
"toolArguments": "Arguments",
|
||||
|
||||
@@ -14,20 +14,37 @@
|
||||
"actions": "Actions"
|
||||
},
|
||||
"noDescription": "No description",
|
||||
"visibility": {
|
||||
"clickToCycle": "Click to change visibility"
|
||||
},
|
||||
"delete": {
|
||||
"title": "Delete Skill",
|
||||
"description": "Are you sure you want to delete \"{{name}}\"? This action cannot be undone.",
|
||||
"confirmLabel": "Delete"
|
||||
},
|
||||
"upload": {
|
||||
"title": "Upload Skill",
|
||||
"description": "Upload a ZIP file containing a SKILL.md at the root with YAML frontmatter (name, description, slug).",
|
||||
"title": "Upload Skills",
|
||||
"description": "Upload ZIP files containing SKILL.md with YAML frontmatter (name, description, slug).",
|
||||
"button": "Upload",
|
||||
"uploading": "Uploading...",
|
||||
"dropOrClick": "Click or drag & drop a .zip file",
|
||||
"dropHere": "Drop .zip file here",
|
||||
"dropOrClick": "Click or drag & drop .zip files",
|
||||
"dropHere": "Drop .zip files here",
|
||||
"onlyZip": "Only .zip files are accepted",
|
||||
"cancel": "Cancel"
|
||||
"cancel": "Cancel",
|
||||
"done": "Done",
|
||||
"tooLarge": "File exceeds 20MB limit",
|
||||
"invalidZip": "Invalid ZIP file",
|
||||
"noSkillMd": "ZIP must contain SKILL.md at root",
|
||||
"emptySkillMd": "SKILL.md is empty",
|
||||
"noFrontmatter": "SKILL.md missing YAML frontmatter",
|
||||
"nameRequired": "name required in SKILL.md frontmatter",
|
||||
"invalidSlug": "Invalid slug format",
|
||||
"failed": "Upload failed",
|
||||
"validating": "Validating...",
|
||||
"validCount": "{{valid}} of {{total}} valid",
|
||||
"uploadCount": "Upload ({{count}})",
|
||||
"successCount": "{{success}} of {{total}} uploaded successfully",
|
||||
"remove": "Remove file"
|
||||
},
|
||||
"edit": {
|
||||
"title": "Edit Skill",
|
||||
|
||||
@@ -76,6 +76,7 @@
|
||||
"selectProvider": "Chọn provider",
|
||||
"sendMessage": "Gửi tin nhắn... (Shift+Enter để xuống dòng)",
|
||||
"sendMessageTitle": "Gửi tin nhắn",
|
||||
"attachFile": "Đính kèm tệp",
|
||||
"skillActivated": "Đã kích hoạt",
|
||||
"skillActivating": "Đang kích hoạt...",
|
||||
"status": "Trạng thái",
|
||||
|
||||
@@ -14,20 +14,37 @@
|
||||
"actions": "Thao tác"
|
||||
},
|
||||
"noDescription": "Không có mô tả",
|
||||
"visibility": {
|
||||
"clickToCycle": "Nhấp để thay đổi hiển thị"
|
||||
},
|
||||
"delete": {
|
||||
"title": "Xóa skill",
|
||||
"description": "Bạn có chắc chắn muốn xóa \"{{name}}\"? Hành động này không thể hoàn tác.",
|
||||
"confirmLabel": "Xóa"
|
||||
},
|
||||
"upload": {
|
||||
"title": "Tải lên skill",
|
||||
"description": "Tải lên tệp ZIP chứa SKILL.md ở thư mục gốc với YAML frontmatter (name, description, slug).",
|
||||
"title": "Tải lên Skill",
|
||||
"description": "Tải lên tệp ZIP chứa SKILL.md với YAML frontmatter (name, description, slug).",
|
||||
"button": "Tải lên",
|
||||
"uploading": "Đang tải lên...",
|
||||
"dropOrClick": "Nhấp hoặc kéo thả tệp .zip",
|
||||
"dropHere": "Thả tệp .zip vào đây",
|
||||
"onlyZip": "Chỉ chấp nhận tệp .zip",
|
||||
"cancel": "Hủy"
|
||||
"cancel": "Hủy",
|
||||
"done": "Xong",
|
||||
"tooLarge": "Tệp vượt quá giới hạn 20MB",
|
||||
"invalidZip": "Tệp ZIP không hợp lệ",
|
||||
"noSkillMd": "ZIP phải chứa SKILL.md ở thư mục gốc",
|
||||
"emptySkillMd": "SKILL.md trống",
|
||||
"noFrontmatter": "SKILL.md thiếu YAML frontmatter",
|
||||
"nameRequired": "Cần có name trong SKILL.md frontmatter",
|
||||
"invalidSlug": "Định dạng slug không hợp lệ",
|
||||
"failed": "Tải lên thất bại",
|
||||
"validating": "Đang xác thực...",
|
||||
"validCount": "{{valid}} / {{total}} hợp lệ",
|
||||
"uploadCount": "Tải lên ({{count}})",
|
||||
"successCount": "{{success}} / {{total}} đã tải lên thành công",
|
||||
"remove": "Xóa tệp"
|
||||
},
|
||||
"edit": {
|
||||
"title": "Chỉnh sửa skill",
|
||||
|
||||
@@ -76,6 +76,7 @@
|
||||
"selectProvider": "选择Provider",
|
||||
"sendMessage": "发送消息... (Shift+Enter 换行)",
|
||||
"sendMessageTitle": "发送消息",
|
||||
"attachFile": "附加文件",
|
||||
"skillActivated": "已激活",
|
||||
"skillActivating": "激活中...",
|
||||
"status": "状态",
|
||||
|
||||
@@ -14,6 +14,9 @@
|
||||
"actions": "操作"
|
||||
},
|
||||
"noDescription": "暂无描述",
|
||||
"visibility": {
|
||||
"clickToCycle": "点击更改可见性"
|
||||
},
|
||||
"delete": {
|
||||
"title": "删除Skill",
|
||||
"description": "确定要删除「{{name}}」吗?此操作无法撤销。",
|
||||
@@ -27,7 +30,21 @@
|
||||
"dropOrClick": "点击或拖放 .zip 文件",
|
||||
"dropHere": "将 .zip 文件拖放到此处",
|
||||
"onlyZip": "只接受 .zip 文件",
|
||||
"cancel": "取消"
|
||||
"cancel": "取消",
|
||||
"done": "完成",
|
||||
"tooLarge": "文件超过 20MB 限制",
|
||||
"invalidZip": "无效的 ZIP 文件",
|
||||
"noSkillMd": "ZIP 必须包含根目录的 SKILL.md",
|
||||
"emptySkillMd": "SKILL.md 为空",
|
||||
"noFrontmatter": "SKILL.md 缺少 YAML frontmatter",
|
||||
"nameRequired": "SKILL.md frontmatter 中需要 name",
|
||||
"invalidSlug": "slug 格式无效",
|
||||
"failed": "上传失败",
|
||||
"validating": "验证中...",
|
||||
"validCount": "{{valid}} / {{total}} 有效",
|
||||
"uploadCount": "上传 ({{count}})",
|
||||
"successCount": "{{success}} / {{total}} 上传成功",
|
||||
"remove": "移除文件"
|
||||
},
|
||||
"edit": {
|
||||
"title": "编辑Skill",
|
||||
|
||||
@@ -7,7 +7,7 @@ import { useIsMobile } from "@/hooks/use-media-query";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ChatSidebar } from "./chat-sidebar";
|
||||
import { ChatThread } from "./chat-thread";
|
||||
import { ChatInput } from "@/components/chat/chat-input";
|
||||
import { ChatInput, type AttachedFile } from "@/components/chat/chat-input";
|
||||
import { useChatSessions } from "./hooks/use-chat-sessions";
|
||||
import { useChatMessages } from "./hooks/use-chat-messages";
|
||||
import { useChatSend } from "./hooks/use-chat-send";
|
||||
@@ -111,7 +111,7 @@ export function ChatPage() {
|
||||
);
|
||||
|
||||
const handleSend = useCallback(
|
||||
(message: string) => {
|
||||
(message: string, files?: AttachedFile[]) => {
|
||||
let key = sessionKey;
|
||||
if (!key) {
|
||||
key = buildNewSessionKey();
|
||||
@@ -119,7 +119,7 @@ export function ChatPage() {
|
||||
navigate(`/chat/${encodeURIComponent(key)}`);
|
||||
}
|
||||
// Pass key directly so send() doesn't use a stale closure value
|
||||
send(message, key);
|
||||
send(message, key, files);
|
||||
setScrollTrigger((n) => n + 1);
|
||||
},
|
||||
[sessionKey, send, buildNewSessionKey, navigate],
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { useState, useCallback } from "react";
|
||||
import { useWs } from "@/hooks/use-ws";
|
||||
import { useWs, useHttp } from "@/hooks/use-ws";
|
||||
import { Methods } from "@/api/protocol";
|
||||
import type { ChatMessage } from "@/types/chat";
|
||||
import type { AttachedFile } from "@/components/chat/chat-input";
|
||||
|
||||
interface UseChatSendOptions {
|
||||
agentId: string;
|
||||
@@ -9,9 +10,15 @@ interface UseChatSendOptions {
|
||||
onExpectRun: () => void;
|
||||
}
|
||||
|
||||
interface MediaUploadResponse {
|
||||
path: string;
|
||||
mime_type: string;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles sending chat messages.
|
||||
* The sessionKey is passed directly to send() to avoid stale closures.
|
||||
* Handles sending chat messages with optional file attachments.
|
||||
* Files are uploaded via HTTP first, then paths are passed to chat.send.
|
||||
*/
|
||||
export function useChatSend({
|
||||
agentId,
|
||||
@@ -19,28 +26,50 @@ export function useChatSend({
|
||||
onExpectRun,
|
||||
}: UseChatSendOptions) {
|
||||
const ws = useWs();
|
||||
const http = useHttp();
|
||||
const [sending, setSending] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const send = useCallback(
|
||||
async (message: string, sessionKey: string) => {
|
||||
if (!ws.isConnected || !message.trim() || !sessionKey) return;
|
||||
async (message: string, sessionKey: string, files?: AttachedFile[]) => {
|
||||
const hasMessage = message.trim().length > 0;
|
||||
const hasFiles = files && files.length > 0;
|
||||
if (!ws.isConnected || (!hasMessage && !hasFiles) || !sessionKey) return;
|
||||
|
||||
const trimmed = message.trim();
|
||||
setError(null);
|
||||
setSending(true);
|
||||
|
||||
// Add user message optimistically
|
||||
// Build optimistic display: show message + file names
|
||||
let displayContent = trimmed;
|
||||
if (hasFiles) {
|
||||
const fileNames = files.map((f) => f.file.name).join(", ");
|
||||
const fileLabel = `[${fileNames}]`;
|
||||
displayContent = displayContent ? `${fileLabel}\n${displayContent}` : fileLabel;
|
||||
}
|
||||
|
||||
onMessageAdded({
|
||||
role: "user",
|
||||
content: trimmed,
|
||||
content: displayContent,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
// Tell the event handler to capture the next run.started for this agent
|
||||
onExpectRun();
|
||||
|
||||
try {
|
||||
// Upload files first, then pass path+filename to chat.send
|
||||
let mediaItems: { path: string; filename: string }[] | undefined;
|
||||
if (hasFiles) {
|
||||
const uploads = await Promise.all(
|
||||
files.map((af) => {
|
||||
const fd = new FormData();
|
||||
fd.append("file", af.file);
|
||||
return http.upload<MediaUploadResponse>("/v1/media/upload", fd);
|
||||
}),
|
||||
);
|
||||
mediaItems = uploads.map((u) => ({ path: u.path, filename: u.filename }));
|
||||
}
|
||||
|
||||
await ws.call<{ runId: string; content: string }>(
|
||||
Methods.CHAT_SEND,
|
||||
{
|
||||
@@ -48,8 +77,9 @@ export function useChatSend({
|
||||
sessionKey,
|
||||
message: trimmed,
|
||||
stream: true,
|
||||
...(mediaItems && { media: mediaItems }),
|
||||
},
|
||||
600_000, // 10 min timeout for long-running agent responses
|
||||
600_000,
|
||||
);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to send message");
|
||||
@@ -57,7 +87,7 @@ export function useChatSend({
|
||||
setSending(false);
|
||||
}
|
||||
},
|
||||
[ws, agentId, onMessageAdded, onExpectRun],
|
||||
[ws, http, agentId, onMessageAdded, onExpectRun],
|
||||
);
|
||||
|
||||
const abort = useCallback(
|
||||
|
||||
@@ -29,7 +29,7 @@ export function StepAgent({ provider, model, onComplete }: StepAgentProps) {
|
||||
const { t } = useTranslation("setup");
|
||||
const { createAgent, deleteAgent, resummonAgent } = useAgents();
|
||||
|
||||
const [displayName] = useState("GoClaw");
|
||||
const [displayName, setDisplayName] = useState("GoClaw");
|
||||
const [agentKey, setAgentKey] = useState("goclaw");
|
||||
const [keyTouched, setKeyTouched] = useState(false);
|
||||
const [description, setDescription] = useState(DEFAULT_PROMPT);
|
||||
@@ -158,8 +158,8 @@ export function StepAgent({ provider, model, onComplete }: StepAgentProps) {
|
||||
</Label>
|
||||
<Input
|
||||
value={displayName}
|
||||
readOnly
|
||||
className="bg-muted cursor-default"
|
||||
onChange={(e) => setDisplayName(e.target.value)}
|
||||
placeholder={t("agent.displayNamePlaceholder", "e.g. GoClaw")}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
/** Client-side validation for skill ZIP files before upload.
|
||||
* Mirrors server-side checks in internal/http/skills_upload.go */
|
||||
import JSZip from "jszip";
|
||||
|
||||
export interface SkillZipValidation {
|
||||
valid: boolean;
|
||||
name?: string;
|
||||
slug?: string;
|
||||
description?: string;
|
||||
/** i18n key under "upload." namespace */
|
||||
error?: string;
|
||||
errorDetail?: string;
|
||||
}
|
||||
|
||||
// Constants matching server-side (internal/http/skills.go)
|
||||
const MAX_SKILL_SIZE = 20 * 1024 * 1024; // 20MB
|
||||
const SLUG_REGEX = /^[a-z0-9][a-z0-9-]*[a-z0-9]$/;
|
||||
const FRONTMATTER_REGEX = /^---\r?\n([\s\S]*?)\r?\n---/;
|
||||
|
||||
/** Validate a skill ZIP file client-side. JSZip is lazy-loaded. */
|
||||
export async function validateSkillZip(file: File): Promise<SkillZipValidation> {
|
||||
if (!file.name.toLowerCase().endsWith(".zip")) {
|
||||
return { valid: false, error: "upload.onlyZip" };
|
||||
}
|
||||
if (file.size > MAX_SKILL_SIZE) {
|
||||
return { valid: false, error: "upload.tooLarge" };
|
||||
}
|
||||
|
||||
let zip: JSZip;
|
||||
try {
|
||||
zip = await JSZip.loadAsync(file);
|
||||
} catch {
|
||||
return { valid: false, error: "upload.invalidZip" };
|
||||
}
|
||||
|
||||
// Find SKILL.md at root or inside single top-level directory
|
||||
const skillMdContent = await findSkillMd(zip);
|
||||
if (skillMdContent === null) {
|
||||
return { valid: false, error: "upload.noSkillMd" };
|
||||
}
|
||||
if (!skillMdContent.trim()) {
|
||||
return { valid: false, error: "upload.emptySkillMd" };
|
||||
}
|
||||
|
||||
// Parse frontmatter
|
||||
const match = skillMdContent.match(FRONTMATTER_REGEX);
|
||||
if (!match?.[1]) {
|
||||
return { valid: false, error: "upload.noFrontmatter" };
|
||||
}
|
||||
const fields = parseFrontmatterFields(match[1]);
|
||||
if (!fields.name) {
|
||||
return { valid: false, error: "upload.nameRequired" };
|
||||
}
|
||||
|
||||
const slug = fields.slug || slugify(fields.name);
|
||||
if (!SLUG_REGEX.test(slug)) {
|
||||
return { valid: false, error: "upload.invalidSlug", errorDetail: slug };
|
||||
}
|
||||
|
||||
return { valid: true, name: fields.name, slug, description: fields.description };
|
||||
}
|
||||
|
||||
/** Find SKILL.md content — root level or inside a single top-level directory */
|
||||
async function findSkillMd(zip: JSZip): Promise<string | null> {
|
||||
// Try root
|
||||
if (zip.files["SKILL.md"] && !zip.files["SKILL.md"].dir) {
|
||||
return zip.files["SKILL.md"].async("string");
|
||||
}
|
||||
// Try single top-level dir (e.g. "my-skill/SKILL.md")
|
||||
const paths = Object.keys(zip.files);
|
||||
const topDirs = new Set(paths.map((p) => p.split("/")[0]).filter(Boolean));
|
||||
for (const dir of topDirs) {
|
||||
const key = dir + "/SKILL.md";
|
||||
if (zip.files[key] && !zip.files[key].dir) {
|
||||
return zip.files[key].async("string");
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Simple key: value parser matching server's parseSkillFrontmatter() */
|
||||
function parseFrontmatterFields(raw: string): Record<string, string> {
|
||||
const fields: Record<string, string> = {};
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const idx = line.indexOf(":");
|
||||
if (idx > 0) {
|
||||
const key = line.slice(0, idx).trim();
|
||||
const val = line
|
||||
.slice(idx + 1)
|
||||
.trim()
|
||||
.replace(/^["']|["']$/g, "");
|
||||
if (key && val) fields[key] = val;
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
function slugify(name: string): string {
|
||||
return name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useState, useRef } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Upload } from "lucide-react";
|
||||
import { Upload, CheckCircle2, XCircle, Loader2, X } from "lucide-react";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
@@ -10,6 +10,18 @@ import {
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { validateSkillZip } from "./lib/validate-skill-zip";
|
||||
|
||||
type FileStatus = "validating" | "valid" | "invalid" | "uploading" | "success" | "error";
|
||||
|
||||
interface FileEntry {
|
||||
id: string;
|
||||
file: File;
|
||||
status: FileStatus;
|
||||
name?: string;
|
||||
slug?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface SkillUploadDialogProps {
|
||||
open: boolean;
|
||||
@@ -19,103 +31,254 @@ interface SkillUploadDialogProps {
|
||||
|
||||
export function SkillUploadDialog({ open, onOpenChange, onUpload }: SkillUploadDialogProps) {
|
||||
const { t } = useTranslation("skills");
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
const [entries, setEntries] = useState<FileEntry[]>([]);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
const [done, setDone] = useState(false);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const addFiles = async (fileList: FileList) => {
|
||||
const newFiles = Array.from(fileList);
|
||||
|
||||
// Build pending list before setState — no side effects inside updater
|
||||
const existingNames = new Set(entries.map((e) => e.file.name));
|
||||
const fresh = newFiles.filter((f) => !existingNames.has(f.name));
|
||||
if (fresh.length === 0) return;
|
||||
|
||||
const pending: FileEntry[] = fresh.map((f) => ({
|
||||
id: crypto.randomUUID(),
|
||||
file: f,
|
||||
status: "validating" as const,
|
||||
}));
|
||||
setEntries((prev) => [...prev, ...pending]);
|
||||
|
||||
// Validate all files concurrently, catch per-file errors
|
||||
const results = await Promise.all(
|
||||
pending.map(async (entry) => {
|
||||
try {
|
||||
return { id: entry.id, result: await validateSkillZip(entry.file) };
|
||||
} catch {
|
||||
return { id: entry.id, result: { valid: false, error: "upload.invalidZip" } as const };
|
||||
}
|
||||
}),
|
||||
);
|
||||
setEntries((prev) =>
|
||||
prev.map((e) => {
|
||||
const match = results.find((r) => r.id === e.id);
|
||||
if (!match) return e;
|
||||
const { result } = match;
|
||||
return {
|
||||
...e,
|
||||
status: result.valid ? "valid" : "invalid",
|
||||
name: "name" in result ? result.name : undefined,
|
||||
slug: "slug" in result ? result.slug : undefined,
|
||||
error: result.error,
|
||||
};
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const removeEntry = (id: string) => {
|
||||
setEntries((prev) => prev.filter((e) => e.id !== id));
|
||||
};
|
||||
|
||||
const handleSubmit = async () => {
|
||||
if (!file) return;
|
||||
setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
await onUpload(file);
|
||||
setFile(null);
|
||||
onOpenChange(false);
|
||||
} catch (err: unknown) {
|
||||
setError(err instanceof Error ? err.message : t("upload.uploading"));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
const validEntries = entries.filter((e) => e.status === "valid");
|
||||
if (validEntries.length === 0) return;
|
||||
setUploading(true);
|
||||
|
||||
for (const entry of validEntries) {
|
||||
setEntries((prev) =>
|
||||
prev.map((e) => (e.id === entry.id ? { ...e, status: "uploading" } : e)),
|
||||
);
|
||||
try {
|
||||
await onUpload(entry.file);
|
||||
setEntries((prev) =>
|
||||
prev.map((e) => (e.id === entry.id ? { ...e, status: "success" } : e)),
|
||||
);
|
||||
} catch (err) {
|
||||
setEntries((prev) =>
|
||||
prev.map((e) =>
|
||||
e.id === entry.id
|
||||
? { ...e, status: "error", error: err instanceof Error ? err.message : t("upload.failed") }
|
||||
: e,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
setUploading(false);
|
||||
setDone(true);
|
||||
};
|
||||
|
||||
const handleClose = (v: boolean) => {
|
||||
if (!loading) {
|
||||
setFile(null);
|
||||
setError("");
|
||||
setDragging(false);
|
||||
onOpenChange(v);
|
||||
}
|
||||
if (uploading) return;
|
||||
setEntries([]);
|
||||
setDragging(false);
|
||||
setDone(false);
|
||||
onOpenChange(v);
|
||||
};
|
||||
|
||||
const acceptDrop = (f: File): boolean => f.name.toLowerCase().endsWith(".zip");
|
||||
|
||||
const handleDrop = (e: React.DragEvent) => {
|
||||
e.preventDefault();
|
||||
setDragging(false);
|
||||
const dropped = e.dataTransfer.files[0];
|
||||
if (dropped && acceptDrop(dropped)) {
|
||||
setFile(dropped);
|
||||
setError("");
|
||||
} else {
|
||||
setError(t("upload.onlyZip"));
|
||||
if (e.dataTransfer.files.length > 0) {
|
||||
addFiles(e.dataTransfer.files);
|
||||
}
|
||||
};
|
||||
|
||||
const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
if (e.target.files && e.target.files.length > 0) {
|
||||
addFiles(e.target.files);
|
||||
}
|
||||
// Reset so same files can be re-selected
|
||||
if (inputRef.current) inputRef.current.value = "";
|
||||
};
|
||||
|
||||
const validCount = entries.filter((e) => e.status === "valid").length;
|
||||
const successCount = entries.filter((e) => e.status === "success").length;
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={handleClose}>
|
||||
<DialogContent>
|
||||
<DialogContent className="max-h-[80dvh] flex flex-col">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{t("upload.title")}</DialogTitle>
|
||||
<DialogDescription>
|
||||
{t("upload.description")}
|
||||
</DialogDescription>
|
||||
<DialogDescription>{t("upload.description")}</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div
|
||||
className={`flex cursor-pointer flex-col items-center gap-2 rounded-md border-2 border-dashed p-8 text-center transition-colors ${
|
||||
dragging
|
||||
? "border-primary bg-primary/5"
|
||||
: "hover:border-primary/50"
|
||||
}`}
|
||||
onClick={() => inputRef.current?.click()}
|
||||
onDragOver={(e) => { e.preventDefault(); setDragging(true); }}
|
||||
onDragEnter={(e) => { e.preventDefault(); setDragging(true); }}
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={handleDrop}
|
||||
>
|
||||
<Upload className="h-8 w-8 text-muted-foreground" />
|
||||
{file ? (
|
||||
<p className="text-sm font-medium">{file.name} ({(file.size / 1024).toFixed(1)} KB)</p>
|
||||
) : (
|
||||
{/* Drop zone — always visible unless uploading/done */}
|
||||
{!uploading && !done && (
|
||||
<div
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
className={`flex cursor-pointer flex-col items-center gap-2 rounded-md border-2 border-dashed p-6 text-center transition-colors ${
|
||||
dragging ? "border-primary bg-primary/5" : "hover:border-primary/50"
|
||||
}`}
|
||||
onClick={() => inputRef.current?.click()}
|
||||
onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") { e.preventDefault(); inputRef.current?.click(); } }}
|
||||
onDragOver={(e) => { e.preventDefault(); setDragging(true); }}
|
||||
onDragEnter={(e) => { e.preventDefault(); setDragging(true); }}
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={handleDrop}
|
||||
>
|
||||
<Upload className="h-8 w-8 text-muted-foreground" />
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{dragging ? t("upload.dropHere") : t("upload.dropOrClick")}
|
||||
</p>
|
||||
)}
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="file"
|
||||
accept=".zip"
|
||||
className="hidden"
|
||||
onChange={(e) => {
|
||||
setFile(e.target.files?.[0] ?? null);
|
||||
setError("");
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="file"
|
||||
accept=".zip"
|
||||
multiple
|
||||
className="hidden"
|
||||
onChange={handleInputChange}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error && <p className="text-sm text-destructive">{error}</p>}
|
||||
{/* File list */}
|
||||
{entries.length > 0 && (
|
||||
<div className="flex flex-col gap-1 overflow-y-auto max-h-[40dvh]">
|
||||
{entries.map((entry) => (
|
||||
<FileEntryRow
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
onRemove={() => removeEntry(entry.id)}
|
||||
uploading={uploading}
|
||||
t={t}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Summary */}
|
||||
{entries.length > 0 && !done && !uploading && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{t("upload.validCount", { valid: validCount, total: entries.length })}
|
||||
</p>
|
||||
)}
|
||||
{done && (
|
||||
<p className="text-sm font-medium text-muted-foreground">
|
||||
{t("upload.successCount", { success: successCount, total: entries.length })}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={() => handleClose(false)} disabled={loading}>
|
||||
<Button variant="outline" onClick={() => handleClose(false)} disabled={uploading}>
|
||||
{t("upload.cancel")}
|
||||
</Button>
|
||||
<Button onClick={handleSubmit} disabled={!file || loading}>
|
||||
{loading ? t("upload.uploading") : t("upload.button")}
|
||||
</Button>
|
||||
{done ? (
|
||||
<Button onClick={() => handleClose(false)}>{t("upload.done")}</Button>
|
||||
) : (
|
||||
<Button onClick={handleSubmit} disabled={validCount === 0 || uploading}>
|
||||
{uploading
|
||||
? t("upload.uploading")
|
||||
: t("upload.uploadCount", { count: validCount })}
|
||||
</Button>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
||||
/** Single file entry row with status icon, name, and remove button */
|
||||
function FileEntryRow({
|
||||
entry,
|
||||
onRemove,
|
||||
uploading,
|
||||
t,
|
||||
}: {
|
||||
entry: FileEntry;
|
||||
onRemove: () => void;
|
||||
uploading: boolean;
|
||||
t: (key: string, opts?: Record<string, unknown>) => string;
|
||||
}) {
|
||||
const sizeKB = (entry.file.size / 1024).toFixed(1);
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2 rounded-md border px-3 py-2 text-sm">
|
||||
<StatusIcon status={entry.status} />
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="truncate font-medium">
|
||||
{entry.name || entry.file.name}
|
||||
</span>
|
||||
<span className="shrink-0 text-xs text-muted-foreground">{sizeKB} KB</span>
|
||||
</div>
|
||||
{entry.status === "invalid" || entry.status === "error" ? (
|
||||
<p className="text-xs text-destructive truncate">{entry.error ? t(entry.error) : t("upload.failed")}</p>
|
||||
) : entry.status === "validating" ? (
|
||||
<p className="text-xs text-muted-foreground">{t("upload.validating")}</p>
|
||||
) : entry.name && entry.status !== "success" ? (
|
||||
<p className="text-xs text-muted-foreground truncate">{entry.file.name}</p>
|
||||
) : null}
|
||||
</div>
|
||||
{/* Remove button — hidden during upload and for completed entries */}
|
||||
{!uploading && entry.status !== "uploading" && entry.status !== "success" && (
|
||||
<button
|
||||
type="button"
|
||||
aria-label={t("upload.remove")}
|
||||
onClick={(e) => { e.stopPropagation(); onRemove(); }}
|
||||
className="shrink-0 rounded-sm p-1 text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<X className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StatusIcon({ status }: { status: FileStatus }) {
|
||||
switch (status) {
|
||||
case "validating":
|
||||
case "uploading":
|
||||
return <Loader2 className="h-4 w-4 shrink-0 animate-spin text-muted-foreground" />;
|
||||
case "valid":
|
||||
return <CheckCircle2 className="h-4 w-4 shrink-0 text-primary" />;
|
||||
case "success":
|
||||
return <CheckCircle2 className="h-4 w-4 shrink-0 text-green-600" />;
|
||||
case "invalid":
|
||||
case "error":
|
||||
return <XCircle className="h-4 w-4 shrink-0 text-destructive" />;
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,14 @@ export function SkillsPage() {
|
||||
refresh();
|
||||
};
|
||||
|
||||
const handleCycleVisibility = async (skill: SkillInfo) => {
|
||||
if (!skill.id) return;
|
||||
const order = ["private", "internal", "public"] as const;
|
||||
const idx = order.indexOf(skill.visibility as typeof order[number]);
|
||||
const next = order[(idx + 1) % order.length];
|
||||
await updateSkill(skill.id, { visibility: next });
|
||||
};
|
||||
|
||||
const handleDelete = async () => {
|
||||
if (!deleteTarget?.id) return;
|
||||
setDeleteLoading(true);
|
||||
@@ -143,9 +151,24 @@ export function SkillsPage() {
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
{skill.visibility && (
|
||||
<Badge variant={visibilityColor[skill.visibility] as "default" | "secondary" | "outline"}>
|
||||
{skill.visibility}
|
||||
</Badge>
|
||||
skill.id ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleCycleVisibility(skill)}
|
||||
title={t("visibility.clickToCycle")}
|
||||
>
|
||||
<Badge
|
||||
variant={visibilityColor[skill.visibility] as "default" | "secondary" | "outline"}
|
||||
className="cursor-pointer hover:opacity-80 transition-opacity"
|
||||
>
|
||||
{skill.visibility}
|
||||
</Badge>
|
||||
</button>
|
||||
) : (
|
||||
<Badge variant={visibilityColor[skill.visibility] as "default" | "secondary" | "outline"}>
|
||||
{skill.visibility}
|
||||
</Badge>
|
||||
)
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
|
||||
Reference in new issue
Block a user