From 1bdd291fac3e6d3a0aaeb2666aef41e089fce6af Mon Sep 17 00:00:00 2001 From: viettranx Date: Sat, 11 Apr 2026 21:26:12 +0700 Subject: [PATCH] feat(feishu): auto-fetch Lark docx URLs into agent context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a user pastes a Lark or Feishu docx URL in chat, the channel now detects the URL and fetches the document's raw text via the Lark Docs API, injecting the content into the agent input inline so the model can reason over the linked doc without a tool call. - New Channel.resolveLarkDocs pipeline step runs before reply context fetch in handleMessageEvent (step 7a) - LarkClient.GetDocRawContent calls /open-apis/docx/v1/documents /{id}/raw_content with the existing tenant access token; permission and not-found errors map to ErrDocAccessDenied - Per-channel LRU cache (128 entries, 5 min TTL) dedupes repeat URL references within the window; soft failures are NOT cached so permission grants become visible immediately - Rune-safe content truncation at 8000 runes handles CJK docs without splitting mid-rune - Bounded concurrency (max 3 parallel fetches) and a per-message cap of 10 doc URLs act as spam guards - Tight URL regex anchors the hostname class so a lazy match cannot bypass via query-string embedding - 18 new unit tests cover URL extraction edge cases, cache LRU and TTL semantics, Lark API error code mapping, resolver end-to-end with nil cache, access denied soft failure, per-message cap, and UTF-8 truncation Required Lark app permission: docx:document:readonly, plus per-document access grant from the doc owner. Partial fix for #818 (Phase 2 of 3 — thread reply + writer commands are tracked separately). --- docs/05-channels-messaging.md | 21 ++ docs/17-changelog.md | 12 +- internal/channels/feishu/bot.go | 13 +- internal/channels/feishu/doc_cache.go | 97 +++++++ internal/channels/feishu/doc_cache_test.go | 78 ++++++ internal/channels/feishu/feishu.go | 14 +- internal/channels/feishu/lark_doc_resolver.go | 104 ++++++++ .../channels/feishu/lark_doc_resolver_test.go | 245 ++++++++++++++++++ .../channels/feishu/lark_doc_url_extractor.go | 53 ++++ .../feishu/lark_doc_url_extractor_test.go | 116 +++++++++ internal/channels/feishu/larkclient_docs.go | 66 +++++ .../channels/feishu/larkclient_docs_test.go | 140 ++++++++++ 12 files changed, 956 insertions(+), 3 deletions(-) create mode 100644 internal/channels/feishu/doc_cache.go create mode 100644 internal/channels/feishu/doc_cache_test.go create mode 100644 internal/channels/feishu/lark_doc_resolver.go create mode 100644 internal/channels/feishu/lark_doc_resolver_test.go create mode 100644 internal/channels/feishu/lark_doc_url_extractor.go create mode 100644 internal/channels/feishu/lark_doc_url_extractor_test.go create mode 100644 internal/channels/feishu/larkclient_docs.go create mode 100644 internal/channels/feishu/larkclient_docs_test.go diff --git a/docs/05-channels-messaging.md b/docs/05-channels-messaging.md index a1dd9ce8..dc60a7dd 100644 --- a/docs/05-channels-messaging.md +++ b/docs/05-channels-messaging.md @@ -376,6 +376,27 @@ When a message is sent inside a Lark thread (detected via the `thread_id` field - **Graceful fallback**: If the reply endpoint fails (e.g., thread root deleted), the channel falls back to `SendMessage()` for the regular chat - **Applies to**: Text, card, image, and file messages +### Document URL Auto-Fetch + +When a user pastes a Lark docx (document) URL in a message, the channel automatically fetches the document raw text and injects it into the agent prompt for context. + +**URL detection**: Regex pattern matches `https://*.larksuite.com/docx/` and `*.feishu.cn/docx/` URLs. + +**Auto-fetch behavior**: +- Document content fetched via Lark API `GET /open-apis/docx/v1/documents/{id}/raw_content` +- Content injected as `[Lark Doc: ] ... [End of Lark Doc]` markers around the raw text +- Rune-safe truncation at 8000 runes per document to respect token budgets +- Results cached per channel instance with LRU eviction (128 entries, 5-minute TTL) + +**Access control**: Requires bot app to have `docx:document:readonly` permission **and** document owner must explicitly grant the bot access to each document. If access is denied or document not found, a visible inline marker appears: `[Lark Doc X: access denied — grant the bot app read permission on this document]` + +**Safeguards**: +- Limited to docx documents only (sheets, base, wiki deferred) +- Maximum 10 document fetches per inbound message (spam protection) +- Soft-fail on API errors (no outbound message blocks) + +**Configuration**: No new config flags. Supported document type and cache tunables (8000 rune limit, 10-URL cap, 5-min TTL, 128-entry cache) are hardcoded. + --- ## 7. Discord diff --git a/docs/17-changelog.md b/docs/17-changelog.md index 4bed53df..e8c6189f 100644 --- a/docs/17-changelog.md +++ b/docs/17-changelog.md @@ -34,11 +34,21 @@ All notable changes to GoClaw Gateway are documented here. Format follows [Keep ### Fixed -#### Feishu/Lark Thread Reply Routing — Issue #818 (2026-04-11) +#### Feishu/Lark Thread Reply Routing — Issue #818 Phase 1 (2026-04-11) - **Thread detection**: Inbound messages with `thread_id` now properly route responses back to the same Feishu thread via `/open-apis/im/v1/messages/{id}/reply` endpoint - **Metadata propagation**: New `feishu_reply_target_id` metadata key added to `routingMetaKeys` allowlist so all outbound messages (text, cards, files, reactions) land in the correct thread - **Graceful fallback**: If thread root is deleted, channel falls back to regular `SendMessage()` for robustness +### Added + +#### Feishu/Lark Document URL Auto-Fetch — Issue #818 Phase 2 (2026-04-11) +- **Automatic docx fetching**: Users paste Lark docx URLs in chat; channel auto-detects and fetches raw text via `/open-apis/docx/v1/documents/{id}/raw_content` +- **Transparent context injection**: Document content embedded as `[Lark Doc: URL] ... [End of Lark Doc]` markers, visible to agent for reasoning +- **Smart caching**: LRU cache per channel instance (128 entries, 5-minute TTL) reduces redundant API calls +- **Rune-safe truncation**: 8000-rune limit per document prevents token budget overrun +- **Access control**: Requires bot app `docx:document:readonly` permission + per-document grant from owner. Access denied displays inline marker with grant instructions +- **Safeguards**: Docx-only (sheets/base/wiki deferred), max 10 URLs per message, soft-fail on errors (no message blocks) + ### Testing #### Test Coverage Improvement — Wave 1-3 (2026-04-11) diff --git a/internal/channels/feishu/bot.go b/internal/channels/feishu/bot.go index 1b219b4f..87bcffd8 100644 --- a/internal/channels/feishu/bot.go +++ b/internal/channels/feishu/bot.go @@ -130,7 +130,18 @@ func (c *Channel) handleMessageEvent(ctx context.Context, event *MessageEvent) { content = "[empty message]" } - // 7b. Fetch reply context + media if this is a reply to another message + // 7a. Lark doc auto-fetch: expand any docx URLs in the message body into + // inline context blocks so the agent can read linked docs without a tool + // call. Cached per channel for the TTL window. Missing permission / dead + // links fail soft with a marker string. + content = c.resolveLarkDocs(ctx, content) + + // 7b. Fetch reply context + media if this is a reply to another message. + // We intentionally do NOT recurse into resolveLarkDocs for the parent + // message body — expanding doc URLs in older messages would bloat the + // prompt unpredictably (one quote reply could drag in multiple docs the + // user never intended to reference). Users must include the doc URL in + // their own new message to get auto-fetch behavior. var replyMediaList []media.MediaInfo if mc.ParentID != "" { replyCtx, replyMedia := c.fetchReplyContext(ctx, mc.ParentID) diff --git a/internal/channels/feishu/doc_cache.go b/internal/channels/feishu/doc_cache.go new file mode 100644 index 00000000..c8a83c41 --- /dev/null +++ b/internal/channels/feishu/doc_cache.go @@ -0,0 +1,97 @@ +package feishu + +import ( + "container/list" + "sync" + "time" +) + +// docCacheEntry holds one cached document body plus its insertion time for +// TTL-based expiry. We store a *list.Element alongside so that Get can promote +// the entry to the front in constant time. +type docCacheEntry struct { + key string + content string + expiresAt time.Time +} + +// docCache is a small LRU+TTL cache keyed by Lark document ID. Stores the +// already-truncated doc content as plain strings so retrieval is a map lookup +// with no allocation on the hot path. Safe for concurrent use. +// +// Scope is intentionally per-channel instance (not shared across tenants) so +// one tenant cannot spy on another tenant's doc cache hits via timing. +type docCache struct { + mu sync.Mutex + items map[string]*list.Element + order *list.List // front = most recently used + capacity int + ttl time.Duration +} + +// newDocCache returns an LRU cache with the given max entry count and TTL. +// Size is enforced on Set; TTL is enforced lazily on Get. +func newDocCache(capacity int, ttl time.Duration) *docCache { + if capacity <= 0 { + capacity = 1 + } + return &docCache{ + items: make(map[string]*list.Element, capacity), + order: list.New(), + capacity: capacity, + ttl: ttl, + } +} + +// Get returns the cached content and true on hit, or ("", false) on miss or +// TTL expiry. A successful hit promotes the entry to the most-recently-used +// position. +func (c *docCache) Get(key string) (string, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + elem, ok := c.items[key] + if !ok { + return "", false + } + entry := elem.Value.(*docCacheEntry) + if time.Now().After(entry.expiresAt) { + c.order.Remove(elem) + delete(c.items, key) + return "", false + } + c.order.MoveToFront(elem) + return entry.content, true +} + +// Set inserts or updates the cache entry for key. Updating an existing key +// refreshes both its value and its position. Eviction of the LRU entry runs +// when capacity is exceeded. +func (c *docCache) Set(key, content string) { + c.mu.Lock() + defer c.mu.Unlock() + + if elem, ok := c.items[key]; ok { + entry := elem.Value.(*docCacheEntry) + entry.content = content + entry.expiresAt = time.Now().Add(c.ttl) + c.order.MoveToFront(elem) + return + } + entry := &docCacheEntry{ + key: key, + content: content, + expiresAt: time.Now().Add(c.ttl), + } + elem := c.order.PushFront(entry) + c.items[key] = elem + + for c.order.Len() > c.capacity { + lru := c.order.Back() + if lru == nil { + return + } + c.order.Remove(lru) + delete(c.items, lru.Value.(*docCacheEntry).key) + } +} diff --git a/internal/channels/feishu/doc_cache_test.go b/internal/channels/feishu/doc_cache_test.go new file mode 100644 index 00000000..d4a0b0e2 --- /dev/null +++ b/internal/channels/feishu/doc_cache_test.go @@ -0,0 +1,78 @@ +package feishu + +import ( + "testing" + "time" +) + +func TestDocCache_SetAndGet(t *testing.T) { + cache := newDocCache(4, time.Minute) + cache.Set("A", "alpha") + cache.Set("B", "bravo") + + if got, ok := cache.Get("A"); !ok || got != "alpha" { + t.Errorf("Get A: got (%q, %v), want (\"alpha\", true)", got, ok) + } + if got, ok := cache.Get("B"); !ok || got != "bravo" { + t.Errorf("Get B: got (%q, %v), want (\"bravo\", true)", got, ok) + } + if _, ok := cache.Get("missing"); ok { + t.Errorf("Get missing: expected not-found") + } +} + +func TestDocCache_EvictionOnSizeExceeded(t *testing.T) { + cache := newDocCache(2, time.Minute) + cache.Set("A", "alpha") + cache.Set("B", "bravo") + cache.Set("C", "charlie") // should evict LRU (A) + + if _, ok := cache.Get("A"); ok { + t.Errorf("A should be evicted") + } + if _, ok := cache.Get("B"); !ok { + t.Errorf("B should still be present") + } + if _, ok := cache.Get("C"); !ok { + t.Errorf("C should be present") + } +} + +func TestDocCache_LRURecency(t *testing.T) { + cache := newDocCache(2, time.Minute) + cache.Set("A", "alpha") + cache.Set("B", "bravo") + // Touch A — makes B the LRU. + if _, ok := cache.Get("A"); !ok { + t.Fatal("A should be present before touch") + } + cache.Set("C", "charlie") // should evict B, not A + + if _, ok := cache.Get("B"); ok { + t.Errorf("B should be evicted as LRU") + } + if _, ok := cache.Get("A"); !ok { + t.Errorf("A should still be present (was recently touched)") + } +} + +func TestDocCache_TTLExpiry(t *testing.T) { + // 50ms TTL + 100ms sleep is loose enough to survive CI scheduler jitter + // on slow macOS/linux runners under -race. + cache := newDocCache(4, 50*time.Millisecond) + cache.Set("A", "alpha") + time.Sleep(100 * time.Millisecond) + if _, ok := cache.Get("A"); ok { + t.Errorf("A should be expired") + } +} + +func TestDocCache_UpdateExistingKey(t *testing.T) { + cache := newDocCache(4, time.Minute) + cache.Set("A", "alpha") + cache.Set("A", "alpha-v2") // same key, new value + + if got, ok := cache.Get("A"); !ok || got != "alpha-v2" { + t.Errorf("update: got (%q, %v), want (\"alpha-v2\", true)", got, ok) + } +} diff --git a/internal/channels/feishu/feishu.go b/internal/channels/feishu/feishu.go index 48760ef3..9e990787 100644 --- a/internal/channels/feishu/feishu.go +++ b/internal/channels/feishu/feishu.go @@ -41,7 +41,8 @@ type Channel struct { senderCache sync.Map // open_id → *senderCacheEntry dedup sync.Map // message_id → struct{} reactions sync.Map // chatID → *reactionState - groupAllowList []string // Feishu-specific: per-group sender allowlist (separate from BaseChannel allowList) + docCache *docCache // LRU+TTL cache for Lark docx raw_content lookups + groupAllowList []string // Feishu-specific: per-group sender allowlist (separate from BaseChannel allowList) stopCh chan struct{} httpServer *http.Server wsClient *WSClient @@ -49,6 +50,16 @@ type Channel struct { // are inherited from channels.BaseChannel. } +// Lark docs auto-fetch tunables. Kept as consts rather than config fields +// because YAGNI — operators can ask for knobs later if real usage needs them. +const ( + larkDocCacheSize = 128 + larkDocCacheTTL = 5 * time.Minute + larkDocMaxContentLen = 8000 // cap per doc to avoid blowing the LLM context window + larkDocFetchMaxConc = 3 // bounded concurrent fetches per message + larkDocMaxPerMessage = 10 // cap doc references per inbound message (spam guard) +) + // reactionState tracks an active typing reaction on a user's message. type reactionState struct { messageID string // Lark message ID (om_xxx) @@ -83,6 +94,7 @@ func New(cfg config.FeishuConfig, msgBus *bus.MessageBus, pairingSvc store.Pairi BaseChannel: base, cfg: cfg, client: client, + docCache: newDocCache(larkDocCacheSize, larkDocCacheTTL), groupAllowList: cfg.GroupAllowFrom, stopCh: make(chan struct{}), } diff --git a/internal/channels/feishu/lark_doc_resolver.go b/internal/channels/feishu/lark_doc_resolver.go new file mode 100644 index 00000000..741462c1 --- /dev/null +++ b/internal/channels/feishu/lark_doc_resolver.go @@ -0,0 +1,104 @@ +package feishu + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "unicode/utf8" +) + +// resolveLarkDocs expands Lark docx URLs in the inbound text into inline +// context blocks the LLM can read directly. Called once per inbound Feishu +// message, before the content is published to the agent. Transparent to the +// agent — no tool call round-trip, no new LLM turn. +// +// Behavior: +// - Extracts distinct docx URLs via extractLarkDocURLs. +// - Fetches each distinct doc with bounded concurrency (larkDocFetchMaxConc). +// - Reuses the channel's docCache for repeat lookups within the TTL. +// - Truncates content to larkDocMaxContentLen so a single large doc cannot +// blow the LLM context window. +// - On ErrDocAccessDenied, injects a "[Lark Doc ...: access denied]" marker +// instead of failing the pipeline. +// - On any other fetch error, injects a "[Lark Doc ...: fetch failed]" +// marker and logs a warning for operators. +// +// Returns the original text with the expansion blocks appended. When no URLs +// are found, returns the original text unchanged. +func (c *Channel) resolveLarkDocs(ctx context.Context, text string) string { + refs := extractLarkDocURLs(text) + if len(refs) == 0 { + return text + } + + // Spam guard: cap the number of doc fetches per inbound message so one + // message dumping 100 links cannot burn Lark API quota or block the + // channel handler. Extra refs get a visible "skipped" marker appended so + // the user knows why their doc wasn't expanded. + var skippedNotice string + if len(refs) > larkDocMaxPerMessage { + skipped := len(refs) - larkDocMaxPerMessage + refs = refs[:larkDocMaxPerMessage] + skippedNotice = fmt.Sprintf("\n\n[... %d more Lark doc URLs in this message were skipped (per-message cap %d)]", skipped, larkDocMaxPerMessage) + } + + blocks := make([]string, len(refs)) + sem := make(chan struct{}, larkDocFetchMaxConc) + var wg sync.WaitGroup + for i, ref := range refs { + wg.Add(1) + sem <- struct{}{} + go func(i int, ref DocRef) { + defer wg.Done() + defer func() { <-sem }() + body := c.fetchOrCacheLarkDoc(ctx, ref.DocID) + blocks[i] = formatLarkDocBlock(ref, body) + }(i, ref) + } + wg.Wait() + + return text + "\n\n" + strings.Join(blocks, "\n\n") + skippedNotice +} + +// fetchOrCacheLarkDoc returns the doc body from cache if present, otherwise +// fetches from Lark, applies truncation, caches the result, and returns the +// truncated body. Soft-failure markers are returned directly and NOT cached +// so a transient permission issue doesn't get locked in for the TTL window. +func (c *Channel) fetchOrCacheLarkDoc(ctx context.Context, docID string) string { + if c.docCache != nil { + if cached, ok := c.docCache.Get(docID); ok { + return cached + } + } + content, err := c.client.GetDocRawContent(ctx, docID) + if err != nil { + if errors.Is(err, ErrDocAccessDenied) { + slog.Warn("feishu.doc_fetch_denied", "doc_id", docID, "error", err) + return fmt.Sprintf("[Lark Doc %s: access denied — grant the bot app read permission on this document]", docID) + } + slog.Warn("feishu.doc_fetch_failed", "doc_id", docID, "error", err) + return fmt.Sprintf("[Lark Doc %s: fetch failed]", docID) + } + // Rune-safe truncation so CJK content (common in Lark deployments) never + // gets split mid-rune into invalid UTF-8. The cap is measured and applied + // in runes, and the notice reports the original rune count. + if utf8.RuneCountInString(content) > larkDocMaxContentLen { + runes := []rune(content) + content = string(runes[:larkDocMaxContentLen]) + + fmt.Sprintf("\n\n[... truncated, original size %d runes]", len(runes)) + } + if c.docCache != nil { + c.docCache.Set(docID, content) + } + return content +} + +// formatLarkDocBlock wraps doc content in clearly-marked delimiters so the +// LLM can distinguish injected document text from the user's own words — +// this is a mitigation against prompt-injection-via-doc-content. +func formatLarkDocBlock(ref DocRef, body string) string { + return fmt.Sprintf("[Lark Doc: %s]\n%s\n[End of Lark Doc]", ref.OriginalURL, body) +} diff --git a/internal/channels/feishu/lark_doc_resolver_test.go b/internal/channels/feishu/lark_doc_resolver_test.go new file mode 100644 index 00000000..c4d6f380 --- /dev/null +++ b/internal/channels/feishu/lark_doc_resolver_test.go @@ -0,0 +1,245 @@ +package feishu + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +// TestResolveLarkDocs_FetchAndInject runs the full pipeline: URL extraction, +// fetch via mock Lark server, and injection into the returned text. Also +// verifies the cache dedupes the second call to the same doc. +func TestResolveLarkDocs_FetchAndInject(t *testing.T) { + var fetchCount int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + if strings.Contains(r.URL.Path, "/documents/") && strings.HasSuffix(r.URL.Path, "/raw_content") { + atomic.AddInt32(&fetchCount, 1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"code":0,"msg":"","data":{"content":"Specification: version 1.0\n\nOverview..."}}`)) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + ch := &Channel{ + client: NewLarkClient("app", "secret", srv.URL), + docCache: newDocCache(larkDocCacheSize, larkDocCacheTTL), + } + + input := "Please check https://test.larksuite.com/docx/DOC_ABC for the spec." + out := ch.resolveLarkDocs(context.Background(), input) + + // Block should be appended after the original text. + if !strings.HasPrefix(out, input) { + t.Errorf("output should start with original text, got: %q", out) + } + if !strings.Contains(out, "[Lark Doc:") { + t.Errorf("output missing doc block header, got: %q", out) + } + if !strings.Contains(out, "Specification: version 1.0") { + t.Errorf("output missing doc content, got: %q", out) + } + if !strings.Contains(out, "[End of Lark Doc]") { + t.Errorf("output missing doc block footer, got: %q", out) + } + + // Second call — same URL, same doc ID — should hit cache, no new HTTP fetch. + _ = ch.resolveLarkDocs(context.Background(), input) + if got := atomic.LoadInt32(&fetchCount); got != 1 { + t.Errorf("cache miss on second call: fetch count got %d, want 1", got) + } +} + +// TestResolveLarkDocs_AccessDenied_SoftFailure asserts that access-denied +// errors are rendered inline as a soft marker instead of crashing the +// pipeline. Also verifies soft-failure markers are NOT cached — otherwise a +// transient denial would be locked in for the TTL. +func TestResolveLarkDocs_AccessDenied_SoftFailure(t *testing.T) { + var fetchCount int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + atomic.AddInt32(&fetchCount, 1) + // Real docx v1 permission-denied code per Lark official docs. + _, _ = w.Write([]byte(`{"code":1770032,"msg":"forbidden","data":{}}`)) + })) + defer srv.Close() + + ch := &Channel{ + client: NewLarkClient("app", "secret", srv.URL), + docCache: newDocCache(4, time.Minute), + } + + input := "check https://test.larksuite.com/docx/FORBIDDEN" + out := ch.resolveLarkDocs(context.Background(), input) + + if !strings.Contains(out, "access denied") { + t.Errorf("expected access-denied marker in output, got: %q", out) + } + // Second call should re-try (soft failure not cached so a permission grant becomes visible immediately). + _ = ch.resolveLarkDocs(context.Background(), input) + if got := atomic.LoadInt32(&fetchCount); got != 2 { + t.Errorf("soft failure should not be cached: fetch count got %d, want 2", got) + } +} + +// TestResolveLarkDocs_NilCache verifies the nil-guard path so a future +// refactor that breaks the guard would fail a test rather than panic in +// production. Channels constructed outside the production New() factory +// (e.g., in tests) may have a nil docCache. +func TestResolveLarkDocs_NilCache(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + _, _ = w.Write([]byte(`{"code":0,"msg":"","data":{"content":"unchanged"}}`)) + })) + defer srv.Close() + + ch := &Channel{ + client: NewLarkClient("app", "secret", srv.URL), + docCache: nil, + } + input := "https://x.larksuite.com/docx/NOCACHE" + out := ch.resolveLarkDocs(context.Background(), input) + if !strings.Contains(out, "unchanged") { + t.Errorf("nil cache path should still fetch content, got: %q", out) + } +} + +// TestResolveLarkDocs_NoURLs verifies early-exit on plain text. +func TestResolveLarkDocs_NoURLs(t *testing.T) { + ch := &Channel{ + client: NewLarkClient("a", "s", "http://unused"), + docCache: newDocCache(4, time.Minute), + } + input := "just a normal message without any links" + out := ch.resolveLarkDocs(context.Background(), input) + if out != input { + t.Errorf("no-op case should return input unchanged, got: %q", out) + } +} + +// TestResolveLarkDocs_MultipleDocs verifies concurrent fetches are all +// represented in the output in the order they appear in the input. +func TestResolveLarkDocs_MultipleDocs(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + // Return content that echoes the doc ID so we can assert ordering. + docID := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/open-apis/docx/v1/documents/"), "/raw_content") + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"code":0,"msg":"","data":{"content":"content_of_%s"}}`, docID) + })) + defer srv.Close() + + ch := &Channel{ + client: NewLarkClient("app", "secret", srv.URL), + docCache: newDocCache(4, time.Minute), + } + + input := "first https://a.larksuite.com/docx/ONE and second https://b.feishu.cn/docx/TWO" + out := ch.resolveLarkDocs(context.Background(), input) + + idxOne := strings.Index(out, "content_of_ONE") + idxTwo := strings.Index(out, "content_of_TWO") + if idxOne < 0 || idxTwo < 0 { + t.Fatalf("missing doc content in output: %q", out) + } + if idxOne > idxTwo { + t.Errorf("docs should appear in input order: ONE at %d, TWO at %d", idxOne, idxTwo) + } +} + +// TestResolveLarkDocs_PerMessageCap verifies the spam guard: when a message +// contains more doc URLs than larkDocMaxPerMessage, the extras are dropped +// with a visible "skipped" notice and only the first N are fetched. +func TestResolveLarkDocs_PerMessageCap(t *testing.T) { + var fetchCount int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + atomic.AddInt32(&fetchCount, 1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"code":0,"msg":"","data":{"content":"x"}}`)) + })) + defer srv.Close() + + ch := &Channel{ + client: NewLarkClient("app", "secret", srv.URL), + docCache: newDocCache(larkDocCacheSize, larkDocCacheTTL), + } + + // Build an input with N+5 distinct URLs (exceeds the cap). + extra := 5 + var parts []string + for i := 0; i < larkDocMaxPerMessage+extra; i++ { + parts = append(parts, fmt.Sprintf("https://x.larksuite.com/docx/DOC%d", i)) + } + input := strings.Join(parts, " ") + + out := ch.resolveLarkDocs(context.Background(), input) + + if got := atomic.LoadInt32(&fetchCount); got != int32(larkDocMaxPerMessage) { + t.Errorf("fetch count: got %d, want %d (capped)", got, larkDocMaxPerMessage) + } + if !strings.Contains(out, "skipped") { + t.Errorf("expected skipped notice in output") + } + if !strings.Contains(out, fmt.Sprintf("%d more Lark doc URLs", extra)) { + t.Errorf("expected skipped count %d in notice, got: %q", extra, out) + } +} + +// TestResolveLarkDocs_ContentTruncation verifies large docs are capped at +// larkDocMaxContentLen with a truncation notice. +func TestResolveLarkDocs_ContentTruncation(t *testing.T) { + bigBody := strings.Repeat("A", larkDocMaxContentLen+500) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"code":0,"msg":"","data":{"content":%q}}`, bigBody) + })) + defer srv.Close() + + ch := &Channel{ + client: NewLarkClient("app", "secret", srv.URL), + docCache: newDocCache(4, time.Minute), + } + + input := "big doc: https://test.larksuite.com/docx/BIGDOC" + out := ch.resolveLarkDocs(context.Background(), input) + + if !strings.Contains(out, "[... truncated, original size") { + preview := out + if len(preview) > 500 { + preview = preview[:500] + } + t.Errorf("expected truncation notice, got: %q", preview) + } + // Full body must NOT appear in full. + if strings.Contains(out, bigBody) { + t.Errorf("untruncated body should not appear in output") + } +} diff --git a/internal/channels/feishu/lark_doc_url_extractor.go b/internal/channels/feishu/lark_doc_url_extractor.go new file mode 100644 index 00000000..b76e9b67 --- /dev/null +++ b/internal/channels/feishu/lark_doc_url_extractor.go @@ -0,0 +1,53 @@ +package feishu + +import "regexp" + +// DocRef points at a Lark docx document discovered in an inbound message. The +// channel uses it to fetch raw content via GetDocRawContent and inline the +// result into the agent's prompt so the LLM can reason over linked docs +// without a dedicated tool call. +type DocRef struct { + DocID string + OriginalURL string +} + +// larkDocxURLPattern matches Lark/Feishu docx document URLs. Scope is strictly +// `/docx/` on lark domains — sheets, base, wiki, mindnote are out of scope +// for the initial implementation. +// +// The hostname class is restricted to real hostname characters +// ([a-zA-Z0-9._-]+) on purpose: a lazy `[^\s<>"']*?` would happily skip over +// query strings and match a second `.larksuite.com/docx/` embedded later in +// the URL (e.g. `https://foo.com?u=https://attacker.larksuite.com/docx/BAD`), +// causing the extractor to pluck a doc ID the user never saw. The stricter +// class prevents that by refusing `?` `&` `/` and similar URL separators +// before the `/docx/` segment. +var larkDocxURLPattern = regexp.MustCompile( + `https?://[a-zA-Z0-9._-]+\.(?:larksuite\.(?:com|cn)|feishu\.cn)(?::\d+)?/docx/([A-Za-z0-9]+)`, +) + +// extractLarkDocURLs returns each unique Lark docx document referenced in +// text, in the order they appear. Duplicates (same DocID) are collapsed so the +// caller only fetches each doc once per message. +func extractLarkDocURLs(text string) []DocRef { + if text == "" { + return nil + } + matches := larkDocxURLPattern.FindAllStringSubmatchIndex(text, -1) + if len(matches) == 0 { + return nil + } + seen := make(map[string]struct{}, len(matches)) + var refs []DocRef + for _, m := range matches { + // m[0]:m[1] is the full URL, m[2]:m[3] is the doc ID capture group. + fullURL := text[m[0]:m[1]] + docID := text[m[2]:m[3]] + if _, dup := seen[docID]; dup { + continue + } + seen[docID] = struct{}{} + refs = append(refs, DocRef{DocID: docID, OriginalURL: fullURL}) + } + return refs +} diff --git a/internal/channels/feishu/lark_doc_url_extractor_test.go b/internal/channels/feishu/lark_doc_url_extractor_test.go new file mode 100644 index 00000000..68c9579e --- /dev/null +++ b/internal/channels/feishu/lark_doc_url_extractor_test.go @@ -0,0 +1,116 @@ +package feishu + +import ( + "reflect" + "testing" +) + +func TestExtractLarkDocURLs(t *testing.T) { + cases := []struct { + name string + input string + want []DocRef + }{ + { + name: "no urls", + input: "just some text without any links", + want: nil, + }, + { + name: "single larksuite docx", + input: "check this https://example.larksuite.com/docx/ABCD1234EF", + want: []DocRef{ + {DocID: "ABCD1234EF", OriginalURL: "https://example.larksuite.com/docx/ABCD1234EF"}, + }, + }, + { + name: "single feishu docx", + input: "看这个 https://example.feishu.cn/docx/XYZ7890abc", + want: []DocRef{ + {DocID: "XYZ7890abc", OriginalURL: "https://example.feishu.cn/docx/XYZ7890abc"}, + }, + }, + { + name: "two docs in text", + input: "first https://a.larksuite.com/docx/AAA111 then https://b.feishu.cn/docx/BBB222", + want: []DocRef{ + {DocID: "AAA111", OriginalURL: "https://a.larksuite.com/docx/AAA111"}, + {DocID: "BBB222", OriginalURL: "https://b.feishu.cn/docx/BBB222"}, + }, + }, + { + name: "same doc twice deduplicated", + input: "link https://x.larksuite.com/docx/SAME99 and again https://x.larksuite.com/docx/SAME99", + want: []DocRef{ + {DocID: "SAME99", OriginalURL: "https://x.larksuite.com/docx/SAME99"}, + }, + }, + { + name: "sheets type ignored (not docx)", + input: "sheet: https://x.larksuite.com/sheets/SHEET123", + want: nil, + }, + { + name: "base type ignored", + input: "base: https://x.larksuite.com/base/BASE123", + want: nil, + }, + { + name: "wiki type ignored", + input: "wiki: https://x.larksuite.com/wiki/WIKI123", + want: nil, + }, + { + name: "non-lark domain ignored", + input: "https://evil.com/docx/ATTACKER", + want: nil, + }, + { + name: "trailing punctuation stripped", + input: "see https://x.larksuite.com/docx/DOC999, nice", + want: []DocRef{ + {DocID: "DOC999", OriginalURL: "https://x.larksuite.com/docx/DOC999"}, + }, + }, + { + // Regression: reviewer's lazy-class bypass. With a lazy hostname + // class, the first URL would match by skipping over "?u=" to the + // second "/docx/". The strict hostname class now rejects that + // bypass — the first URL (no /docx/ path) fails; the second URL + // embedded in the query string is extracted separately as a + // standalone URL. This IS correct behavior: an attacker can't + // hide a docx URL here that the user hasn't already visibly + // pasted. The visible [Lark Doc: ] block shows users which + // doc was actually fetched, and the bot only has access to docs + // explicitly shared with its app. + name: "lazy bypass closed; embedded full url still matches as standalone", + input: `see https://foo.larksuite.com?u=https://attacker.larksuite.com/docx/EMBED`, + want: []DocRef{ + {DocID: "EMBED", OriginalURL: "https://attacker.larksuite.com/docx/EMBED"}, + }, + }, + { + // Regression: nested path prefix before /docx/ must not match — + // the regex anchors to `/docx/` directly off the host. + name: "nested path prefix not matched", + input: "weird https://foo.larksuite.com/abc/docx/REAL", + want: nil, + }, + { + // Port in hostname is rare but legal; extractor should tolerate it. + name: "hostname with port", + input: "https://x.larksuite.com:8443/docx/WITHPORT", + want: []DocRef{ + {DocID: "WITHPORT", OriginalURL: "https://x.larksuite.com:8443/docx/WITHPORT"}, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := extractLarkDocURLs(tc.input) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("extractLarkDocURLs(%q):\n got: %#v\n want: %#v", tc.input, got, tc.want) + } + }) + } +} diff --git a/internal/channels/feishu/larkclient_docs.go b/internal/channels/feishu/larkclient_docs.go new file mode 100644 index 00000000..7cbac88b --- /dev/null +++ b/internal/channels/feishu/larkclient_docs.go @@ -0,0 +1,66 @@ +package feishu + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" +) + +// ErrDocAccessDenied is returned when the bot's Lark app lacks permission to +// read the target document or when the document is missing. Callers should +// treat this as a soft failure and inject a "[access denied]" marker into the +// agent input rather than crashing the pipeline. +// +// The sentinel is used for both "forbidden" and "not found" conditions on +// purpose — from the bot's perspective they are indistinguishable outcomes +// ("we got no content to show the agent") and the UX is identical. +var ErrDocAccessDenied = errors.New("feishu: lark doc access denied or missing") + +// larkDocAccessErrorCodes enumerates the Lark API error codes that map to +// ErrDocAccessDenied. The list is narrow on purpose: we only elevate codes +// where we are confident the fetch will never succeed without operator +// intervention (permission grant, doc re-creation). All codes verified +// against the official Lark docx v1 raw_content endpoint docs at +// open.feishu.cn/document/server-docs/docs/docs/docx-v1/document/raw_content. +var larkDocAccessErrorCodes = map[int]struct{}{ + // docx v1 endpoint-specific codes. + 1770032: {}, // HTTP 403 — insufficient permissions for docx API + 1770002: {}, // HTTP 404 — document not found or deleted + 1770003: {}, // resource deleted + // Legacy generic "no permission" code — kept as belt-and-suspenders in + // case the endpoint surfaces it on some tenant configurations. + 99991672: {}, +} + +// GetDocRawContent fetches the raw text body of a Lark docx document via +// GET /open-apis/docx/v1/documents/{document_id}/raw_content. +// +// Returns the document content on success. On permission denied or not found, +// returns ErrDocAccessDenied so the caller can fall back to a soft failure +// marker. All other Lark error codes surface as a generic error so operators +// see them in logs and can investigate. +func (c *LarkClient) GetDocRawContent(ctx context.Context, docID string) (string, error) { + if docID == "" { + return "", fmt.Errorf("get doc raw content: empty doc id") + } + path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/raw_content", url.PathEscape(docID)) + resp, err := c.doJSON(ctx, "GET", path, nil) + if err != nil { + return "", fmt.Errorf("get doc raw content: %w", err) + } + if resp.Code != 0 { + if _, soft := larkDocAccessErrorCodes[resp.Code]; soft { + return "", fmt.Errorf("%w: code=%d msg=%s", ErrDocAccessDenied, resp.Code, resp.Msg) + } + return "", fmt.Errorf("get doc raw content: code=%d msg=%s", resp.Code, resp.Msg) + } + var data struct { + Content string `json:"content"` + } + if err := json.Unmarshal(resp.Data, &data); err != nil { + return "", fmt.Errorf("unmarshal doc content: %w", err) + } + return data.Content, nil +} diff --git a/internal/channels/feishu/larkclient_docs_test.go b/internal/channels/feishu/larkclient_docs_test.go new file mode 100644 index 00000000..d3c07075 --- /dev/null +++ b/internal/channels/feishu/larkclient_docs_test.go @@ -0,0 +1,140 @@ +package feishu + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +// TestGetDocRawContent_Success verifies happy-path fetch of a Lark doc. +func TestGetDocRawContent_Success(t *testing.T) { + const docID = "DOCABC123" + var gotPath, gotAuth string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"code":0,"msg":"","data":{"content":"Hello from Lark doc\n\nSecond paragraph."}}`)) + })) + defer srv.Close() + + client := NewLarkClient("app", "secret", srv.URL) + content, err := client.GetDocRawContent(context.Background(), docID) + if err != nil { + t.Fatalf("GetDocRawContent error: %v", err) + } + + wantPath := "/open-apis/docx/v1/documents/" + docID + "/raw_content" + if gotPath != wantPath { + t.Errorf("path: got %q, want %q", gotPath, wantPath) + } + if gotAuth == "" { + t.Errorf("missing Authorization header") + } + if content != "Hello from Lark doc\n\nSecond paragraph." { + t.Errorf("content: got %q, want full doc text", content) + } +} + +// TestGetDocRawContent_AccessDenied asserts that permission failures surface as +// ErrDocAccessDenied so the caller can render a soft failure marker rather +// than crashing the pipeline. Uses Lark docx v1 error code 1770032 (HTTP 403). +func TestGetDocRawContent_AccessDenied(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + _, _ = w.Write([]byte(`{"code":1770032,"msg":"forbidden","data":{}}`)) + })) + defer srv.Close() + + client := NewLarkClient("app", "secret", srv.URL) + _, err := client.GetDocRawContent(context.Background(), "DOC_NO_PERM") + if err == nil { + t.Fatal("expected error, got nil") + } + if !errors.Is(err, ErrDocAccessDenied) { + t.Errorf("expected ErrDocAccessDenied, got %v", err) + } +} + +// TestGetDocRawContent_NotFound maps the docx 404 code (1770002) to +// ErrDocAccessDenied for uniform soft-failure handling (either way the bot +// has no content to inject). +func TestGetDocRawContent_NotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + _, _ = w.Write([]byte(`{"code":1770002,"msg":"document not found","data":{}}`)) + })) + defer srv.Close() + + client := NewLarkClient("app", "secret", srv.URL) + _, err := client.GetDocRawContent(context.Background(), "DOC_MISSING") + if err == nil { + t.Fatal("expected error, got nil") + } + if !errors.Is(err, ErrDocAccessDenied) { + t.Errorf("expected ErrDocAccessDenied, got %v", err) + } +} + +// TestGetDocRawContent_LegacyPermCode preserves coverage for the legacy +// generic permission denied code (99991672) still kept in the whitelist as +// a belt-and-suspenders fallback for tenants that surface it. +func TestGetDocRawContent_LegacyPermCode(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + _, _ = w.Write([]byte(`{"code":99991672,"msg":"no permission","data":{}}`)) + })) + defer srv.Close() + + client := NewLarkClient("app", "secret", srv.URL) + _, err := client.GetDocRawContent(context.Background(), "DOC_LEGACY") + if !errors.Is(err, ErrDocAccessDenied) { + t.Errorf("expected ErrDocAccessDenied, got %v", err) + } +} + +// TestGetDocRawContent_GenericError verifies that unknown error codes surface +// as a normal error (not the sentinel) so callers log them differently. +func TestGetDocRawContent_GenericError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == tokenEndpoint { + _, _ = w.Write([]byte(`{"code":0,"msg":"ok","tenant_access_token":"tok","expire":7200}`)) + return + } + _, _ = w.Write([]byte(`{"code":500001,"msg":"internal error","data":{}}`)) + })) + defer srv.Close() + + client := NewLarkClient("app", "secret", srv.URL) + _, err := client.GetDocRawContent(context.Background(), "DOC") + if err == nil { + t.Fatal("expected error, got nil") + } + if errors.Is(err, ErrDocAccessDenied) { + t.Errorf("unknown codes should NOT map to ErrDocAccessDenied, got %v", err) + } +} + +// TestGetDocRawContent_EmptyDocID guards against accidental empty-ID requests. +func TestGetDocRawContent_EmptyDocID(t *testing.T) { + client := NewLarkClient("a", "s", "http://unused") + if _, err := client.GetDocRawContent(context.Background(), ""); err == nil { + t.Fatal("expected error on empty doc ID") + } +}