fix(vision): file-ref mode + media pipeline fixes for image visibility (#511)

* fix(vision): file-ref mode + media pipeline fixes for image visibility

Replace inline base64 image loading with file path references when
read_image provider is configured. LLM calls read_image(path=...)
instead of receiving 25K-250K tokens of base64 per image.

Changes:
- Agent loop: skip base64 context storage in file-ref mode, only
  load historical images for inline fallback
- New enrichImagePaths() enriches ALL user messages with file paths
  (not just current turn) so historical images are accessible
- System prompt: imperative tool summaries ("REQUIRED when you see
  <media:image> tags") + dedicated "Media Files" section
- Slack: store mediaPaths in pending history + CollectMedia on mention
- Feishu: add CollectMedia for group context history media
- RC-1 fix: save enriched content (with media IDs/paths) to DB
  instead of raw req.Message

Closes #509

* fix(vision): resolve Claude review — Feishu early media + skip loadImages in file-ref mode

- Feishu: download media BEFORE mention gate (step 4) and store file
  paths in pending history via Media field. Reuse early-resolved
  MediaInfo at step 10 to avoid double-download. CollectMedia now
  returns actual paths instead of empty.
- Agent loop: guard loadImages() behind !deferToReadImageTool so
  file-ref mode avoids unnecessary disk I/O + base64 encoding.
This commit is contained in:
Duy /zuey/ authored and GitHub committed 2026-03-27 14:20:05 +07:00
1 parent 4dafd70c83
commit 231bc9684e
6 files changed
+159 -33

No files matched your search

+33 -11
View File
@@ -60,14 +60,32 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
// 3. Resolve sender name (cached)
senderName := c.resolveSenderName(ctx, mc.SenderID)
// 4. Group policy
// 4. Resolve media BEFORE mention gate so non-mentioned messages
// also have their files downloaded and stored in pending history.
var earlyMedia []media.MediaInfo
switch mc.ContentType {
case "image", "file", "audio", "video", "sticker":
earlyMedia = c.resolveMediaFromMessage(ctx, mc.MessageID, mc.ContentType, msg.Content)
case "post":
if imageKeys := extractPostImageKeys(msg.Content); len(imageKeys) > 0 {
earlyMedia = c.resolvePostImages(ctx, mc.MessageID, imageKeys)
}
}
var earlyMediaPaths []string
for _, m := range earlyMedia {
if m.FilePath != "" {
earlyMediaPaths = append(earlyMediaPaths, m.FilePath)
}
}
// 5. Group policy
if mc.ChatType == "group" {
if !c.checkGroupPolicy(ctx, mc.SenderID, mc.ChatID) {
slog.Debug("feishu group message rejected by policy", "sender_id", mc.SenderID, "chat_id", mc.ChatID)
return
}
// 5. RequireMention check — record to history if not mentioned
// 6. RequireMention check — record to history if not mentioned
requireMention := true
if c.cfg.RequireMention != nil {
requireMention = *c.cfg.RequireMention
@@ -81,6 +99,7 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
Sender: senderName,
SenderID: mc.SenderID,
Body: mc.Content,
Media: earlyMediaPaths,
Timestamp: time.Now(),
MessageID: messageID,
}, c.historyLimit)
@@ -173,23 +192,26 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
}
}
// 10. Resolve inbound media (image, file, audio, video, sticker)
// 10. Build media list from early-resolved media (step 4) + reply media.
// Media was already downloaded before the mention gate — reuse results.
var mediaList []media.MediaInfo
// Reply media first (context), current media second.
// Reply media first (context), current-message media second.
if len(replyMediaList) > 0 {
mediaList = append(mediaList, replyMediaList...)
}
switch mc.ContentType {
case "image", "file", "audio", "video", "sticker":
mediaList = append(mediaList, c.resolveMediaFromMessage(ctx, mc.MessageID, mc.ContentType, msg.Content)...)
case "post":
if imageKeys := extractPostImageKeys(msg.Content); len(imageKeys) > 0 {
mediaList = append(mediaList, c.resolvePostImages(ctx, mc.MessageID, imageKeys)...)
mediaList = append(mediaList, earlyMedia...)
// 10b. Collect media from pending history (files downloaded by earlier non-mentioned messages).
var mediaFiles []bus.MediaFile
if mc.ChatType == "group" && c.historyLimit > 0 {
if histMediaPaths := c.groupHistory.CollectMedia(chatID); len(histMediaPaths) > 0 {
for _, p := range histMediaPaths {
mediaFiles = append(mediaFiles, bus.MediaFile{Path: p}) // cannot use append(slice, other...) — different types
}
}
}
// 11. Process media: STT transcription, document extraction, build tags
var mediaFiles []bus.MediaFile
if len(mediaList) > 0 {
var extraContent string
for i := range mediaList {