mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-11 03:13:24 +00:00
feat(channels): extract media from replied-to messages across all channels
When a user replies to a message containing images/attachments and mentions the bot, the agent now receives media from both the replied-to message and the current message. Reply media is tagged with "(from replied message)" so the LLM can distinguish origin. Changes per channel: - Telegram: always merge reply + current media (was fallback-only) - Discord: extract ReferencedMessage content + attachments, implicit mention - Feishu: extend fetchReplyContext to download parent media via API - Slack: add fetchThreadParentContext for thread parent files - Shared: FromReply field on MediaInfo, annotated BuildMediaTags
This commit is contained in:
1 parent
7301ad5eb4
commit
f8571dec88
7 files changed
+181
-36
No files matched your search
@@ -70,6 +70,21 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
|
||||
// Build content
|
||||
content := m.Content
|
||||
|
||||
// Build reply context if replying to another message.
|
||||
if m.ReferencedMessage != nil {
|
||||
author := "unknown"
|
||||
if m.ReferencedMessage.Author != nil {
|
||||
author = m.ReferencedMessage.Author.Username
|
||||
}
|
||||
body := channels.Truncate(m.ReferencedMessage.Content, 500)
|
||||
replyCtx := fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", author, body)
|
||||
if content != "" {
|
||||
content = replyCtx + "\n\n" + content
|
||||
} else {
|
||||
content = replyCtx
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve media attachments (download files, classify types)
|
||||
maxBytes := c.config.MediaMaxBytes
|
||||
if maxBytes <= 0 {
|
||||
@@ -77,6 +92,15 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
|
||||
}
|
||||
mediaList := resolveMedia(m.Attachments, maxBytes)
|
||||
|
||||
// Download media from replied-to message and merge (reply first, current second).
|
||||
if m.ReferencedMessage != nil && len(m.ReferencedMessage.Attachments) > 0 {
|
||||
replyMedia := resolveMedia(m.ReferencedMessage.Attachments, maxBytes)
|
||||
for i := range replyMedia {
|
||||
replyMedia[i].FromReply = true
|
||||
}
|
||||
mediaList = append(replyMedia, mediaList...)
|
||||
}
|
||||
|
||||
// Process media: STT, document extraction, build tags
|
||||
var mediaFiles []bus.MediaFile
|
||||
if len(mediaList) > 0 {
|
||||
@@ -143,6 +167,12 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
|
||||
break
|
||||
}
|
||||
}
|
||||
// Reply to bot's message counts as implicit mention.
|
||||
if !mentioned && m.ReferencedMessage != nil &&
|
||||
m.ReferencedMessage.Author != nil &&
|
||||
m.ReferencedMessage.Author.ID == c.botUserID {
|
||||
mentioned = true
|
||||
}
|
||||
if !mentioned {
|
||||
// Collect media file paths for group history context.
|
||||
var mediaPaths []string
|
||||
|
||||
@@ -110,11 +110,14 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
|
||||
content = "[empty message]"
|
||||
}
|
||||
|
||||
// 7b. Fetch reply context if this is a reply to another message
|
||||
// 7b. Fetch reply context + media if this is a reply to another message
|
||||
var replyMediaList []media.MediaInfo
|
||||
if mc.ParentID != "" {
|
||||
if replyCtx := c.fetchReplyContext(ctx, mc.ParentID); replyCtx != "" {
|
||||
replyCtx, replyMedia := c.fetchReplyContext(ctx, mc.ParentID)
|
||||
if replyCtx != "" {
|
||||
content += "\n\n" + replyCtx
|
||||
}
|
||||
replyMediaList = replyMedia
|
||||
}
|
||||
|
||||
// 8. Topic session
|
||||
@@ -172,9 +175,13 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
|
||||
|
||||
// 10. Resolve inbound media (image, file, audio, video, sticker)
|
||||
var mediaList []media.MediaInfo
|
||||
// Reply media first (context), current media second.
|
||||
if len(replyMediaList) > 0 {
|
||||
mediaList = append(mediaList, replyMediaList...)
|
||||
}
|
||||
switch mc.ContentType {
|
||||
case "image", "file", "audio", "video", "sticker":
|
||||
mediaList = c.resolveMediaFromMessage(ctx, mc.MessageID, mc.ContentType, msg.Content)
|
||||
mediaList = append(mediaList, c.resolveMediaFromMessage(ctx, mc.MessageID, mc.ContentType, msg.Content)...)
|
||||
}
|
||||
|
||||
// 11. Process media: STT transcription, document extraction, build tags
|
||||
@@ -269,22 +276,19 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) {
|
||||
const replyContextMaxLen = 500
|
||||
|
||||
// fetchReplyContext fetches the parent message content and returns a formatted
|
||||
// reply context string, similar to Telegram's [Replying to ...] format.
|
||||
func (c *Channel) fetchReplyContext(ctx context.Context, parentID string) string {
|
||||
// reply context string + any downloaded media from the parent message.
|
||||
func (c *Channel) fetchReplyContext(ctx context.Context, parentID string) (string, []media.MediaInfo) {
|
||||
resp, err := c.client.GetMessage(ctx, parentID)
|
||||
if err != nil {
|
||||
slog.Debug("feishu: failed to fetch parent message", "parent_id", parentID, "error", err)
|
||||
return ""
|
||||
return "", nil
|
||||
}
|
||||
if len(resp.Items) == 0 {
|
||||
return ""
|
||||
return "", nil
|
||||
}
|
||||
|
||||
item := &resp.Items[0]
|
||||
body := parseMessageContent(item.Body.Content, item.MsgType)
|
||||
if body == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Resolve sender name
|
||||
senderName := "unknown"
|
||||
@@ -294,6 +298,26 @@ func (c *Channel) fetchReplyContext(ctx context.Context, parentID string) string
|
||||
}
|
||||
}
|
||||
|
||||
body = channels.Truncate(body, replyContextMaxLen)
|
||||
return fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", senderName, body)
|
||||
// Build reply context text.
|
||||
var replyCtx string
|
||||
if body != "" {
|
||||
body = channels.Truncate(body, replyContextMaxLen)
|
||||
replyCtx = fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", senderName, body)
|
||||
}
|
||||
|
||||
// Download media from parent message (image, file, audio, video, sticker).
|
||||
var replyMedia []media.MediaInfo
|
||||
switch item.MsgType {
|
||||
case "image", "file", "audio", "video", "sticker":
|
||||
replyMedia = c.resolveMediaFromMessage(ctx, parentID, item.MsgType, item.Body.Content)
|
||||
for i := range replyMedia {
|
||||
replyMedia[i].FromReply = true
|
||||
}
|
||||
if len(replyMedia) > 0 {
|
||||
slog.Debug("feishu: resolved media from replied message",
|
||||
"parent_id", parentID, "media_count", len(replyMedia))
|
||||
}
|
||||
}
|
||||
|
||||
return replyCtx, replyMedia
|
||||
}
|
||||
@@ -23,4 +23,5 @@ type MediaInfo struct {
|
||||
FileName string // original filename
|
||||
FileSize int64
|
||||
Transcript string // STT transcript for audio/voice (empty if not transcribed)
|
||||
FromReply bool // true if media came from a replied-to/quoted message
|
||||
}
|
||||
@@ -13,33 +13,42 @@ const docMaxChars = 200_000
|
||||
|
||||
// BuildMediaTags generates content tags for media items (matching TS media placeholder format).
|
||||
// For audio/voice items that have been transcribed, the transcript is embedded in a <transcript> block.
|
||||
// Items with FromReply=true are annotated with "(from replied message)" so the LLM can distinguish
|
||||
// media from the current message vs media from the message being replied to.
|
||||
func BuildMediaTags(mediaList []MediaInfo) string {
|
||||
var tags []string
|
||||
for _, m := range mediaList {
|
||||
var tag string
|
||||
switch m.Type {
|
||||
case TypeImage:
|
||||
tags = append(tags, "<media:image>")
|
||||
tag = "<media:image>"
|
||||
case TypeVideo, TypeAnimation:
|
||||
tags = append(tags, "<media:video>")
|
||||
tag = "<media:video>"
|
||||
case TypeAudio:
|
||||
if m.Transcript != "" {
|
||||
tags = append(tags, fmt.Sprintf("<media:audio>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript)))
|
||||
tag = fmt.Sprintf("<media:audio>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript))
|
||||
} else {
|
||||
tags = append(tags, "<media:audio>")
|
||||
tag = "<media:audio>"
|
||||
}
|
||||
case TypeVoice:
|
||||
if m.Transcript != "" {
|
||||
tags = append(tags, fmt.Sprintf("<media:voice>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript)))
|
||||
tag = fmt.Sprintf("<media:voice>\n<transcript>%s</transcript>", html.EscapeString(m.Transcript))
|
||||
} else {
|
||||
tags = append(tags, "<media:voice>")
|
||||
tag = "<media:voice>"
|
||||
}
|
||||
case TypeDocument:
|
||||
if m.FileName != "" {
|
||||
tags = append(tags, fmt.Sprintf("<media:document name=%q>", m.FileName))
|
||||
tag = fmt.Sprintf("<media:document name=%q>", m.FileName)
|
||||
} else {
|
||||
tags = append(tags, "<media:document>")
|
||||
tag = "<media:document>"
|
||||
}
|
||||
}
|
||||
if tag != "" {
|
||||
if m.FromReply {
|
||||
tag += " (from replied message)"
|
||||
}
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
}
|
||||
return strings.Join(tags, "\n")
|
||||
}
|
||||
|
||||
@@ -108,17 +108,41 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
|
||||
|
||||
// Process file attachments from Slack message
|
||||
var mediaPaths []string
|
||||
var allItems []mediaItem
|
||||
if ev.Message != nil && len(ev.Message.Files) > 0 {
|
||||
items, docContent := c.resolveMedia(ev.Message.Files)
|
||||
allItems = append(allItems, items...)
|
||||
|
||||
for _, item := range items {
|
||||
if item.FilePath != "" {
|
||||
mediaPaths = append(mediaPaths, item.FilePath)
|
||||
if docContent != "" {
|
||||
if content != "" {
|
||||
content = content + "\n\n" + docContent
|
||||
} else {
|
||||
content = docContent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prepend media tags and document content to message text
|
||||
mediaTags := buildMediaTags(items)
|
||||
// Fetch reply context + media from thread parent message.
|
||||
// Only when replying in a thread (ThreadTimeStamp != TimeStamp).
|
||||
threadTS := ev.ThreadTimeStamp
|
||||
if threadTS != "" && threadTS != ev.TimeStamp {
|
||||
replyCtx, replyItems := c.fetchThreadParentContext(context.Background(), channelID, threadTS)
|
||||
if replyCtx != "" {
|
||||
if content != "" {
|
||||
content = replyCtx + "\n\n" + content
|
||||
} else {
|
||||
content = replyCtx
|
||||
}
|
||||
}
|
||||
// Reply media first (context), current media second.
|
||||
if len(replyItems) > 0 {
|
||||
allItems = append(replyItems, allItems...)
|
||||
}
|
||||
}
|
||||
|
||||
// Build media tags and collect file paths from all items.
|
||||
if len(allItems) > 0 {
|
||||
mediaTags := buildMediaTags(allItems)
|
||||
if mediaTags != "" {
|
||||
if content != "" {
|
||||
content = mediaTags + "\n\n" + content
|
||||
@@ -126,11 +150,9 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
|
||||
content = mediaTags
|
||||
}
|
||||
}
|
||||
if docContent != "" {
|
||||
if content != "" {
|
||||
content = content + "\n\n" + docContent
|
||||
} else {
|
||||
content = docContent
|
||||
for _, item := range allItems {
|
||||
if item.FilePath != "" {
|
||||
mediaPaths = append(mediaPaths, item.FilePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -141,7 +163,6 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
|
||||
|
||||
// Determine local_key and thread context
|
||||
localKey := channelID
|
||||
threadTS := ev.ThreadTimeStamp
|
||||
if !isDM && threadTS != "" {
|
||||
localKey = fmt.Sprintf("%s:thread:%s", channelID, threadTS)
|
||||
}
|
||||
@@ -256,3 +277,49 @@ func (c *Channel) handleMessage(ev *slackevents.MessageEvent) {
|
||||
c.groupHistory.Clear(localKey)
|
||||
}
|
||||
}
|
||||
|
||||
// fetchThreadParentContext fetches the thread-start message and returns a formatted
|
||||
// reply context string + any downloaded media from the parent message.
|
||||
func (c *Channel) fetchThreadParentContext(ctx context.Context, channelID, threadTS string) (string, []mediaItem) {
|
||||
params := &slackapi.GetConversationHistoryParameters{
|
||||
ChannelID: channelID,
|
||||
Latest: threadTS,
|
||||
Limit: 1,
|
||||
Inclusive: true,
|
||||
}
|
||||
history, err := c.api.GetConversationHistoryContext(ctx, params)
|
||||
if err != nil || len(history.Messages) == 0 {
|
||||
slog.Debug("slack: failed to fetch thread parent", "channel", channelID, "thread_ts", threadTS, "error", err)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
parent := &history.Messages[0]
|
||||
|
||||
// Build reply context text.
|
||||
var replyCtx string
|
||||
if parent.Text != "" {
|
||||
body := channels.Truncate(parent.Text, 500)
|
||||
userName := c.resolveDisplayName(parent.User)
|
||||
replyCtx = fmt.Sprintf("[Replying to %s]\n%s\n[/Replying]", userName, body)
|
||||
}
|
||||
|
||||
// Download files from parent message.
|
||||
var replyItems []mediaItem
|
||||
if len(parent.Files) > 0 {
|
||||
items, docContent := c.resolveMedia(parent.Files)
|
||||
for i := range items {
|
||||
items[i].FromReply = true
|
||||
}
|
||||
replyItems = items
|
||||
// Append extracted document text to reply context.
|
||||
if docContent != "" {
|
||||
if replyCtx != "" {
|
||||
replyCtx += "\n\n" + docContent
|
||||
} else {
|
||||
replyCtx = docContent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return replyCtx, replyItems
|
||||
}
|
||||
@@ -18,6 +18,7 @@ type mediaItem struct {
|
||||
FilePath string // local temp file path
|
||||
FileName string // original filename
|
||||
ContentType string // MIME type
|
||||
FromReply bool // true if media came from a replied-to/thread-parent message
|
||||
}
|
||||
|
||||
// resolveMedia downloads and classifies files attached to a Slack message.
|
||||
@@ -83,16 +84,24 @@ func classifyMime(mime string) string {
|
||||
}
|
||||
|
||||
// buildMediaTags generates content tags for media items.
|
||||
// Items with FromReply=true are annotated so the LLM can distinguish origin.
|
||||
func buildMediaTags(items []mediaItem) string {
|
||||
var tags []string
|
||||
for _, m := range items {
|
||||
var tag string
|
||||
switch m.Type {
|
||||
case "image":
|
||||
tags = append(tags, "<media:image>")
|
||||
tag = "<media:image>"
|
||||
case "audio":
|
||||
tags = append(tags, "<media:audio>")
|
||||
tag = "<media:audio>"
|
||||
case "document":
|
||||
tags = append(tags, fmt.Sprintf("<media:document file=%q>", m.FileName))
|
||||
tag = fmt.Sprintf("<media:document file=%q>", m.FileName)
|
||||
}
|
||||
if tag != "" {
|
||||
if m.FromReply {
|
||||
tag += " (from replied message)"
|
||||
}
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
}
|
||||
return strings.Join(tags, "\n")
|
||||
|
||||
@@ -314,10 +314,15 @@ func (c *Channel) handleMessage(ctx context.Context, update telego.Update) {
|
||||
// Deferred until after mention + pairing gates to avoid downloading
|
||||
// media for messages that only get recorded in pending history.
|
||||
mediaList, mediaErrors := c.resolveMedia(ctx, message)
|
||||
if message.ReplyToMessage != nil && len(mediaList) == 0 {
|
||||
if message.ReplyToMessage != nil {
|
||||
replyMedia, replyErrors := c.resolveMedia(ctx, message.ReplyToMessage)
|
||||
if len(replyMedia) > 0 {
|
||||
mediaList = append(mediaList, replyMedia...)
|
||||
// Tag reply media so LLM knows which images came from the replied-to message.
|
||||
for i := range replyMedia {
|
||||
replyMedia[i].FromReply = true
|
||||
}
|
||||
// Reply media first (context), current media second.
|
||||
mediaList = append(replyMedia, mediaList...)
|
||||
slog.Debug("telegram: resolved media from replied message",
|
||||
"reply_msg_id", message.ReplyToMessage.MessageID,
|
||||
"media_count", len(replyMedia),
|
||||
|
||||
Reference in new issue
Block a user