feat(channels): extract media from replied-to messages across all channels

When a user replies to a message containing images/attachments and
mentions the bot, the agent now receives media from both the replied-to
message and the current message. Reply media is tagged with
"(from replied message)" so the LLM can distinguish origin.

Changes per channel:
- Telegram: always merge reply + current media (was fallback-only)
- Discord: extract ReferencedMessage content + attachments, implicit mention
- Feishu: extend fetchReplyContext to download parent media via API
- Slack: add fetchThreadParentContext for thread parent files
- Shared: FromReply field on MediaInfo, annotated BuildMediaTags
This commit is contained in:
Duy Nguyen committed 2026-03-22 16:38:36 +07:00
1 parent 7301ad5eb4
commit f8571dec88
7 files changed
+181 -36

No files matched your search

+30
View File
@@ -70,6 +70,21 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
// Build content
content := m.Content
// Build reply context if replying to another message.
if m.ReferencedMessage != nil {
author := "unknown"
if m.ReferencedMessage.Author != nil {
author = m.ReferencedMessage.Author.Username
}
body := channels.Truncate(m.ReferencedMessage.Content, 500)
replyCtx := fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", author, body)
if content != "" {
content = replyCtx + "\n\n" + content
} else {
content = replyCtx
}
}
// Resolve media attachments (download files, classify types)
maxBytes := c.config.MediaMaxBytes
if maxBytes <= 0 {
@@ -77,6 +92,15 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
}
mediaList := resolveMedia(m.Attachments, maxBytes)
// Download media from replied-to message and merge (reply first, current second).
if m.ReferencedMessage != nil && len(m.ReferencedMessage.Attachments) > 0 {
replyMedia := resolveMedia(m.ReferencedMessage.Attachments, maxBytes)
for i := range replyMedia {
replyMedia[i].FromReply = true
}
mediaList = append(replyMedia, mediaList...)
}
// Process media: STT, document extraction, build tags
var mediaFiles []bus.MediaFile
if len(mediaList) > 0 {
@@ -143,6 +167,12 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
break
}
}
// Reply to bot's message counts as implicit mention.
if !mentioned && m.ReferencedMessage != nil &&
m.ReferencedMessage.Author != nil &&
m.ReferencedMessage.Author.ID == c.botUserID {
mentioned = true
}
if !mentioned {
// Collect media file paths for group history context.
var mediaPaths []string
+36 -12
View File
@@ -110,11 +110,14 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
content = "[empty message]"
}
// 7b. Fetch reply context if this is a reply to another message
// 7b. Fetch reply context + media if this is a reply to another message
var replyMediaList []media.MediaInfo
if mc.ParentID != "" {
if replyCtx := c.fetchReplyContext(ctx, mc.ParentID); replyCtx != "" {
replyCtx, replyMedia := c.fetchReplyContext(ctx, mc.ParentID)
if replyCtx != "" {
content += "\n\n" + replyCtx
}
replyMediaList = replyMedia
}
// 8. Topic session
@@ -172,9 +175,13 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
// 10. Resolve inbound media (image, file, audio, video, sticker)
var mediaList []media.MediaInfo
// Reply media first (context), current media second.
if len(replyMediaList) > 0 {
mediaList = append(mediaList, replyMediaList...)
}
switch mc.ContentType {
case "image", "file", "audio", "video", "sticker":
mediaList = c.resolveMediaFromMessage(ctx, mc.MessageID, mc.ContentType, msg.Content)
mediaList = append(mediaList, c.resolveMediaFromMessage(ctx, mc.MessageID, mc.ContentType, msg.Content)...)
}
// 11. Process media: STT transcription, document extraction, build tags
@@ -269,22 +276,19 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
const replyContextMaxLen = 500
// fetchReplyContext fetches the parent message content and returns a formatted
// reply context string, similar to Telegram's [Replying to ...] format.
func (c *Channel) fetchReplyContext(ctx context.Context, parentID string) string {
// reply context string + any downloaded media from the parent message.
func (c *Channel) fetchReplyContext(ctx context.Context, parentID string) (string, []media.MediaInfo) {
resp, err := c.client.GetMessage(ctx, parentID)
if err != nil {
slog.Debug("feishu: failed to fetch parent message", "parent_id", parentID, "error", err)
return ""
return "", nil
}
if len(resp.Items) == 0 {
return ""
return "", nil
}
item := &resp.Items[0]
body := parseMessageContent(item.Body.Content, item.MsgType)
if body == "" {
return ""
}
// Resolve sender name
senderName := "unknown"
@@ -294,6 +298,26 @@ func (c *Channel) fetchReplyContext(ctx context.Context, parentID string) string
}
}
body = channels.Truncate(body, replyContextMaxLen)
return fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", senderName, body)
// Build reply context text.
var replyCtx string
if body != "" {
body = channels.Truncate(body, replyContextMaxLen)
replyCtx = fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", senderName, body)
}
// Download media from parent message (image, file, audio, video, sticker).
var replyMedia []media.MediaInfo
switch item.MsgType {
case "image", "file", "audio", "video", "sticker":
replyMedia = c.resolveMediaFromMessage(ctx, parentID, item.MsgType, item.Body.Content)
for i := range replyMedia {
replyMedia[i].FromReply = true
}
if len(replyMedia) > 0 {
slog.Debug("feishu: resolved media from replied message",
"parent_id", parentID, "media_count", len(replyMedia))
}
}
return replyCtx, replyMedia
}
+1
View File
@@ -23,4 +23,5 @@ type MediaInfo struct {
FileName string // original filename
FileSize int64
Transcript string // STT transcript for audio/voice (empty if not transcribed)
FromReply bool // true if media came from a replied-to/quoted message
}
+17 -8
View File
@@ -13,33 +13,42 @@ const docMaxChars = 200_000
// BuildMediaTags generates content tags for media items (matching TS media placeholder format).
// For audio/voice items that have been transcribed, the transcript is embedded in a <transcript> block.
// Items with FromReply=true are annotated with "(from replied message)" so the LLM can distinguish
// media from the current message vs media from the message being replied to.
func BuildMediaTags(mediaList []MediaInfo) string {
var tags []string
for _, m := range mediaList {
var tag string
switch m.Type {
case TypeImage:
tags = append(tags, "<media:image>")
tag = "<media:image>"
case TypeVideo, TypeAnimation:
tags = append(tags, "<media:video>")
tag = "<media:video>"
case TypeAudio:
if m.Transcript != "" {
tags = append(tags, fmt.Sprintf("<media:audio>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript)))
tag = fmt.Sprintf("<media:audio>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript))
} else {
tags = append(tags, "<media:audio>")
tag = "<media:audio>"
}
case TypeVoice:
if m.Transcript != "" {
tags = append(tags, fmt.Sprintf("<media:voice>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript)))
tag = fmt.Sprintf("<media:voice>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript))
} else {
tags = append(tags, "<media:voice>")
tag = "<media:voice>"
}
case TypeDocument:
if m.FileName != "" {
tags = append(tags, fmt.Sprintf("<media:document name=%q>", m.FileName))
tag = fmt.Sprintf("<media:document name=%q>", m.FileName)
} else {
tags = append(tags, "<media:document>")
tag = "<media:document>"
}
}
if tag != "" {
if m.FromReply {
tag += " (from replied message)"
}
tags = append(tags, tag)
}
}
return strings.Join(tags, "\n")
}
+78 -11
View File
@@ -108,17 +108,41 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
// Process file attachments from Slack message
var mediaPaths []string
var allItems []mediaItem
if ev.Message != nil && len(ev.Message.Files) > 0 {
items, docContent := c.resolveMedia(ev.Message.Files)
allItems = append(allItems, items...)
for _, item := range items {
if item.FilePath != "" {
mediaPaths = append(mediaPaths, item.FilePath)
if docContent != "" {
if content != "" {
content = content + "\n\n" + docContent
} else {
content = docContent
}
}
}
// Prepend media tags and document content to message text
mediaTags := buildMediaTags(items)
// Fetch reply context + media from thread parent message.
// Only when replying in a thread (ThreadTimeStamp != TimeStamp).
threadTS := ev.ThreadTimeStamp
if threadTS != "" && threadTS != ev.TimeStamp {
replyCtx, replyItems := c.fetchThreadParentContext(context.Background(), channelID, threadTS)
if replyCtx != "" {
if content != "" {
content = replyCtx + "\n\n" + content
} else {
content = replyCtx
}
}
// Reply media first (context), current media second.
if len(replyItems) > 0 {
allItems = append(replyItems, allItems...)
}
}
// Build media tags and collect file paths from all items.
if len(allItems) > 0 {
mediaTags := buildMediaTags(allItems)
if mediaTags != "" {
if content != "" {
content = mediaTags + "\n\n" + content
@@ -126,11 +150,9 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
content = mediaTags
}
}
if docContent != "" {
if content != "" {
content = content + "\n\n" + docContent
} else {
content = docContent
for _, item := range allItems {
if item.FilePath != "" {
mediaPaths = append(mediaPaths, item.FilePath)
}
}
}
@@ -141,7 +163,6 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
// Determine local_key and thread context
localKey := channelID
threadTS := ev.ThreadTimeStamp
if !isDM && threadTS != "" {
localKey = fmt.Sprintf("%s:thread:%s", channelID, threadTS)
}
@@ -256,3 +277,49 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
c.groupHistory.Clear(localKey)
}
}
// fetchThreadParentContext fetches the thread-start message and returns a formatted
// reply context string + any downloaded media from the parent message.
func (c *Channel) fetchThreadParentContext(ctx context.Context, channelID, threadTS string) (string, []mediaItem) {
params := &slackapi.GetConversationHistoryParameters{
ChannelID: channelID,
Latest: threadTS,
Limit: 1,
Inclusive: true,
}
history, err := c.api.GetConversationHistoryContext(ctx, params)
if err != nil || len(history.Messages) == 0 {
slog.Debug("slack: failed to fetch thread parent", "channel", channelID, "thread_ts", threadTS, "error", err)
return "", nil
}
parent := &history.Messages[0]
// Build reply context text.
var replyCtx string
if parent.Text != "" {
body := channels.Truncate(parent.Text, 500)
userName := c.resolveDisplayName(parent.User)
replyCtx = fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", userName, body)
}
// Download files from parent message.
var replyItems []mediaItem
if len(parent.Files) > 0 {
items, docContent := c.resolveMedia(parent.Files)
for i := range items {
items[i].FromReply = true
}
replyItems = items
// Append extracted document text to reply context.
if docContent != "" {
if replyCtx != "" {
replyCtx += "\n\n" + docContent
} else {
replyCtx = docContent
}
}
}
return replyCtx, replyItems
}
+12 -3
View File
@@ -18,6 +18,7 @@ type mediaItem struct {
FilePath string // local temp file path
FileName string // original filename
ContentType string // MIME type
FromReply bool // true if media came from a replied-to/thread-parent message
}
// resolveMedia downloads and classifies files attached to a Slack message.
@@ -83,16 +84,24 @@ func classifyMime(mime string) string {
}
// buildMediaTags generates content tags for media items.
// Items with FromReply=true are annotated so the LLM can distinguish origin.
func buildMediaTags(items []mediaItem) string {
var tags []string
for _, m := range items {
var tag string
switch m.Type {
case "image":
tags = append(tags, "<media:image>")
tag = "<media:image>"
case "audio":
tags = append(tags, "<media:audio>")
tag = "<media:audio>"
case "document":
tags = append(tags, fmt.Sprintf("<media:document file=%q>", m.FileName))
tag = fmt.Sprintf("<media:document file=%q>", m.FileName)
}
if tag != "" {
if m.FromReply {
tag += " (from replied message)"
}
tags = append(tags, tag)
}
}
return strings.Join(tags, "\n")
+7 -2
View File
@@ -314,10 +314,15 @@ func (c *Channel) handleMessage(ctx context.Context, update telego.Update) {
// Deferred until after mention + pairing gates to avoid downloading
// media for messages that only get recorded in pending history.
mediaList, mediaErrors := c.resolveMedia(ctx, message)
if message.ReplyToMessage != nil && len(mediaList) == 0 {
if message.ReplyToMessage != nil {
replyMedia, replyErrors := c.resolveMedia(ctx, message.ReplyToMessage)
if len(replyMedia) > 0 {
mediaList = append(mediaList, replyMedia...)
// Tag reply media so LLM knows which images came from the replied-to message.
for i := range replyMedia {
replyMedia[i].FromReply = true
}
// Reply media first (context), current media second.
mediaList = append(replyMedia, mediaList...)
slog.Debug("telegram: resolved media from replied message",
"reply_msg_id", message.ReplyToMessage.MessageID,
"media_count", len(replyMedia),