package agent import ( "context" "log/slog" "os" "path/filepath" "strings" "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/tools" ) // isTextMime returns true for MIME types representing human-readable text content. // Covers text/* family plus common application/* types that are text-based. func isTextMime(mime string) bool { if strings.HasPrefix(mime, "text/") { return true } switch mime { case "application/json", "application/xml", "application/yaml", "application/x-yaml", "application/javascript": return true } return false } // collectRefsByKind gathers MediaRefs of a given kind from message history // in chronological order, then appends current-turn refs. The last ref is the // newest document for read_document's omitted media_id fallback. func collectRefsByKind(messages []providers.Message, currentRefs []providers.MediaRef, kind string) []providers.MediaRef { var refs []providers.MediaRef for i := range messages { for _, ref := range messages[i].MediaRefs { if ref.Kind == kind { refs = append(refs, ref) } } } for _, ref := range currentRefs { if ref.Kind == kind { refs = append(refs, ref) } } return refs } // enrichInputMedia processes incoming media (images, documents, audio, video), // persists them, enriches messages with media tags, and populates context // with refs for tool access. Returns updated context, modified messages, and current-turn media refs. func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages []providers.Message) (context.Context, []providers.Message, []providers.MediaRef) { // 1b. Determine image routing strategy. // If read_image tool has a dedicated vision provider, images are NOT attached inline // to the main LLM — the agent calls read_image tool instead. This avoids sending // images to providers that don't support vision or have strict content filters. deferToReadImageTool := l.hasReadImageProvider() if !deferToReadImageTool { // Inline mode: reload historical images directly into messages for main provider. l.reloadMediaForMessages(messages, maxMediaReloadMessages) } // 2. Process media: sanitize images, persist to media store. var mediaRefs []providers.MediaRef if len(req.Media) > 0 { mediaRefs = l.persistMedia(req.SessionKey, req.Media, tools.ToolWorkspaceFromCtx(ctx)) if req.RunKind == "delegate" { rehomeDelegatedMediaMessage(messages, req.Media, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) } // Register persisted text uploads in vault (async, non-blocking). if l.onTextUploaded != nil { for _, ref := range mediaRefs { if ref.Path != "" && isTextMime(ref.MimeType) { if content, err := os.ReadFile(ref.Path); err == nil { go l.onTextUploaded(context.WithoutCancel(ctx), ref.Path, string(content)) } } } } // Load current-turn images from persisted refs (Path is always set for new uploads). var imageFiles []bus.MediaFile for _, ref := range mediaRefs { if ref.Kind == "image" && ref.Path != "" { imageFiles = append(imageFiles, bus.MediaFile{Path: ref.Path, MimeType: ref.MimeType, Filename: filepath.Base(ref.Path)}) } } if deferToReadImageTool { // File-ref mode: images primarily accessed via read_image(path=...). // Still load into context as fallback — if LLM omits the path param, // read_image can fall back to context images. This costs Go memory // but NOT LLM tokens (base64 is in Go context, not sent to provider). if images := loadImages(imageFiles); len(images) > 0 { ctx = tools.WithMediaImages(ctx, images) } slog.Info("vision: file-ref mode, images accessible via read_image tool", "count", len(imageFiles), "agent", l.id) } else if images := loadImages(imageFiles); len(images) > 0 { // Inline mode: read files, base64 encode, attach to message + context. messages[len(messages)-1].Images = images ctx = tools.WithMediaImages(ctx, images) slog.Info("vision: attached images inline to main provider", "count", len(images), "agent", l.id) } } // 2a. Load historical images into context for read_image tool. // Both modes need this: inline mode for main LLM, file-ref mode as fallback // when LLM calls read_image without the path param. if l.mediaStore != nil { ctx = l.loadHistoricalImagesForTool(ctx, mediaRefs, messages) } // 2b. Collect image MediaRefs (historical + current) for exact media_id // resolution by read_image and create_image. if imageRefs := collectRefsByKind(messages, mediaRefs, "image"); len(imageRefs) > 0 { ctx = tools.WithMediaImageRefs(ctx, imageRefs) } // 2c. Collect document MediaRefs (historical + current) for read_document tool. if docRefs := collectRefsByKind(messages, mediaRefs, "document"); len(docRefs) > 0 { ctx = tools.WithMediaDocRefs(ctx, docRefs) // Enrich the last user message with exact IDs and logical workspace paths. // Only current-turn refs are paired with current-turn tags. l.enrichDocumentPaths(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) } // 2d. Collect audio MediaRefs (historical + current) for read_audio tool. if audioRefs := collectRefsByKind(messages, mediaRefs, "audio"); len(audioRefs) > 0 { ctx = tools.WithMediaAudioRefs(ctx, audioRefs) l.enrichAudioIDs(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) } // 2e. Collect video MediaRefs (historical + current) for read_video tool. if videoRefs := collectRefsByKind(messages, mediaRefs, "video"); len(videoRefs) > 0 { ctx = tools.WithMediaVideoRefs(ctx, videoRefs) l.enrichVideoIDs(messages, mediaRefs) } // 2f. Enrich tags with persisted media IDs so the LLM // knows images were received and stored (consistent with audio/video enrichment). l.enrichImageIDs(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) // 2f-ii. Enrich ALL user messages' image tags with logical file paths. // This upgrades legacy absolute paths and keeps current and historical media // references safe regardless of whether images are also attached inline. l.enrichImagePaths(messages, tools.ToolWorkspaceFromCtx(ctx)) // 2g. Collect all media file paths for team workspace auto-collect. // When the leader calls team_tasks(create), these paths are copied to the // team workspace so members can access attached files. if len(mediaRefs) > 0 { var mediaPaths []string for _, ref := range mediaRefs { // Prefer workspace-local path (.uploads/) over canonical .media/ path. if ref.Path != "" { mediaPaths = append(mediaPaths, ref.Path) } else if l.mediaStore != nil { p, err := l.mediaStore.LoadPath(ref.ID) if err != nil { continue } mediaPaths = append(mediaPaths, p) } } if len(mediaPaths) > 0 { ctx = tools.WithRunMediaPaths(ctx, mediaPaths) // Extract original filenames from tags // in the last user message (enriched in step 2b above). if lastMsg := messages[len(messages)-1]; lastMsg.Role == "user" { if nameMap := tools.ExtractMediaNameMap(lastMsg.Content); len(nameMap) > 0 { ctx = tools.WithRunMediaNames(ctx, nameMap) } } } } return ctx, messages, mediaRefs } // rehomeDelegatedMediaMessage replaces caller-owned attachment paths with the // delegatee-owned persisted copies and ensures every imported attachment has a // media tag. The normal enrichment pass then adds the new IDs and local paths. func rehomeDelegatedMediaMessage( messages []providers.Message, input []bus.MediaFile, refs []providers.MediaRef, workspace string, ) { if len(messages) == 0 || len(refs) == 0 { return } lastUser := -1 for i := len(messages) - 1; i >= 0; i-- { if messages[i].Role == "user" { lastUser = i break } } if lastUser < 0 { return } content := messages[lastUser].Content for i := 0; i < len(input) && i < len(refs); i++ { if input[i].Path != "" && refs[i].Path != "" { replacement := logicalWorkspaceMediaPath(workspace, refs[i].Path) if replacement == "" { replacement = filepath.Base(refs[i].Path) } content = strings.ReplaceAll(content, input[i].Path, replacement) } } tagCounts := map[string]int{ "image": strings.Count(content, " 0 { tagCounts[ref.Kind]-- continue } if content != "" && !strings.HasSuffix(content, "\n") { content += "\n" } content += tag } messages[lastUser].Content = content }