fix(web_fetch): increase read limit and add empty content detection

- Read limit: maxChars*4 → max(maxChars*10, 512KB) to handle pages with
  large <head> sections (WordPress sites often have 30-50KB+ heads)
- Add warning message when HTML extraction returns empty despite non-empty
  response body (bot protection, JS-only pages)
- Enable HTTP/2 via ForceAttemptHTTP2 on custom Transport

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
viettranxandClaude Opus 4.6 committed 2026-03-03 18:05:03 +07:00
1 parent dbd317ac81
commit 25be8cc41b
1 file changed
+8 -2
+8 -2
View File
@@ -140,6 +140,7 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
client := &http.Client{
Timeout: time.Duration(fetchTimeoutSeconds) * time.Second,
Transport: &http.Transport{
ForceAttemptHTTP2: true,
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
TLSHandshakeTimeout: 15 * time.Second,
@@ -163,8 +164,9 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
}
defer resp.Body.Close()
// Limit body reading to avoid memory issues
limitReader := io.LimitReader(resp.Body, int64(maxChars*4)) // read extra for HTML overhead
// Read enough HTML to reach <body> content — pages often have 30-50KB+ <head> sections.
readLimit := int64(max(maxChars*10, 512*1024))
limitReader := io.LimitReader(resp.Body, readLimit)
body, err := io.ReadAll(limitReader)
if err != nil {
return "", fmt.Errorf("read body: %w", err)
@@ -196,6 +198,10 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
text = htmlToText(string(body))
extractor = "html-to-text"
}
if text == "" && len(body) > 0 {
text = "[No content extracted. The page may require JavaScript to render, " +
"or returned a bot-protection challenge. Try using browser automation instead.]"
}
default:
text = string(body)