From 25be8cc41bf1734e0b61e70aeadb1b994d91b9d6 Mon Sep 17 00:00:00 2001 From: viettranx Date: Tue, 3 Mar 2026 18:05:03 +0700 Subject: [PATCH] fix(web_fetch): increase read limit and add empty content detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Read limit: maxChars*4 → max(maxChars*10, 512KB) to handle pages with large sections (WordPress sites often have 30-50KB+ heads) - Add warning message when HTML extraction returns empty despite non-empty response body (bot protection, JS-only pages) - Enable HTTP/2 via ForceAttemptHTTP2 on custom Transport Co-Authored-By: Claude Opus 4.6 --- internal/tools/web_fetch.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/tools/web_fetch.go b/internal/tools/web_fetch.go index 2e5eb2c7..e05acb0e 100644 --- a/internal/tools/web_fetch.go +++ b/internal/tools/web_fetch.go @@ -140,6 +140,7 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, client := &http.Client{ Timeout: time.Duration(fetchTimeoutSeconds) * time.Second, Transport: &http.Transport{ + ForceAttemptHTTP2: true, MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, TLSHandshakeTimeout: 15 * time.Second, @@ -163,8 +164,9 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, } defer resp.Body.Close() - // Limit body reading to avoid memory issues - limitReader := io.LimitReader(resp.Body, int64(maxChars*4)) // read extra for HTML overhead + // Read enough HTML to reach content — pages often have 30-50KB+ sections. + readLimit := int64(max(maxChars*10, 512*1024)) + limitReader := io.LimitReader(resp.Body, readLimit) body, err := io.ReadAll(limitReader) if err != nil { return "", fmt.Errorf("read body: %w", err) @@ -196,6 +198,10 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, text = htmlToText(string(body)) extractor = "html-to-text" } + if text == "" && len(body) > 0 { + text = "[No content extracted. The page may require JavaScript to render, " + + "or returned a bot-protection challenge. Try using browser automation instead.]" + } default: text = string(body)